Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(endpoint): stop the service before rollback restores its state file

shukieshah24fa1777b3bb
diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
index f751638b..b38779d4 100644
--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
@@ -132,7 +132,7 @@ type fileSnapshot struct {
// service manager. service.Manager satisfies this as-is.
type serviceController interface {
Unload() error
- Restart() error
+ Load() error
}
type installRollback struct {
@@ -173,25 +173,38 @@ func (r *installRollback) Rollback(manifest Manifest) {
rollback(manifest)
return
}
- // Only tear down a service this install brought up. One that was already running belongs to the
- // previous, working install, and taking it down is not this transaction's to do.
- if r.ServiceLoaded && !r.ServiceWasRunning {
+ // Stop first, always, and while the service state on disk still describes the process this
+ // install started.
+ //
+ // Ordering is the whole of it. The supervised backend's "unit file" *is* its pidfile, and that
+ // file is one of the tracked files below -- so restoring it puts back the pid from before the
+ // install, which by then names a process that has already been terminated. Unloading after the
+ // restore would therefore look up a dead pid, stop nothing, and leave the collector this failed
+ // install started running on the new config, holding the ports, with a second one spawned
+ // beside it.
+ if r.ServiceLoaded {
_ = r.Manager.Unload()
}
+
restoreBackups(manifest.Backups)
for i := len(r.files) - 1; i >= 0; i-- {
path := r.files[i]
restoreFile(path, r.snapshots[path])
}
- // Restart after the files are back, so the collector comes up on the restored configuration
- // rather than the one this failed attempt wrote. Ordering matters: the collector reads its
- // config at startup, so restarting before the restore would leave it running the very config
- // that failed.
+
+ // Bring back a service that was running before this install, now that the configuration it was
+ // running is back on disk. A collector reads its config at startup, so this has to come after
+ // the restore to be worth doing at all.
+ //
+ // Without it a failed reinstall of a healthy endpoint would leave the machine with the service
+ // stopped and, on systemd, disabled -- surviving reboot. The upgrade would not merely fail, it
+ // would end collection. A service this install created has no such claim: nothing was running
+ // before, and leaving a half-installed endpoint registered is worse than leaving it absent.
//
- // Best-effort. If the restart does not take, the endpoint is no worse off than the failure that
+ // Best-effort. If it does not come back, the endpoint is no worse off than the failure that
// brought us here, and the install error the caller receives is the more useful signal.
if r.ServiceLoaded && r.ServiceWasRunning {
- _ = r.Manager.Restart()
+ _ = r.Manager.Load()
}
}
diff --git a/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go b/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go
index cedfa9d8..307f98f6 100644
--- a/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go
+++ b/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go
@@ -1,37 +1,49 @@
package lifecycle
import (
+ "os"
+ "path/filepath"
"strings"
"testing"
)
-// recordingService notes what rollback asked of the service manager, in order.
-type recordingService struct{ calls []string }
+// recordingService notes what rollback asked of the service manager, in order, interleaved with
+// the file restores so the sequence can be asserted rather than just the set of calls.
+type recordingService struct {
+ calls *[]string
+}
+
+func (r recordingService) Unload() error {
+ *r.calls = append(*r.calls, "unload")
+ return nil
+}
-func (r *recordingService) Unload() error { r.calls = append(r.calls, "unload"); return nil }
-func (r *recordingService) Restart() error { r.calls = append(r.calls, "restart"); return nil }
+func (r recordingService) Load() error {
+ *r.calls = append(*r.calls, "load")
+ return nil
+}
// Rollback has to distinguish a service this install started from one that was already running.
//
-// Unloading is right for the first: leaving a half-installed endpoint registered is worse than
-// removing it. It is wrong for the second. A reinstall over a healthy endpoint that fails partway
-// -- readiness timing out, a harness config erroring -- would otherwise stop *and disable* the
-// collector that was working before the attempt, and on systemd that disable survives reboot. The
-// upgrade would not merely fail; it would take collection down with it.
-func TestRollbackDoesNotDisableAServiceItFoundRunning(t *testing.T) {
+// Unloading and leaving it that way is right for the first: nothing was running before, and a
+// half-installed endpoint left registered is worse than one left absent. It is wrong for the
+// second. A reinstall over a healthy endpoint that fails partway would otherwise leave the
+// collector stopped and, on systemd, disabled -- surviving reboot. The upgrade would not merely
+// fail, it would end collection.
+func TestRollbackRestoresAServiceItFoundRunning(t *testing.T) {
for name, tc := range map[string]struct {
loaded, wasRunning bool
want []string
}{
"started by this install": {loaded: true, wasRunning: false, want: []string{"unload"}},
- "already running before it": {loaded: true, wasRunning: true, want: []string{"restart"}},
+ "already running before it": {loaded: true, wasRunning: true, want: []string{"unload", "load"}},
"never got as far as the service": {loaded: false, wasRunning: true, want: nil},
"nothing loaded, nothing running": {loaded: false, wasRunning: false, want: nil},
} {
t.Run(name, func(t *testing.T) {
- svc := &recordingService{}
+ var calls []string
r := &installRollback{
- Manager: svc,
+ Manager: recordingService{calls: &calls},
ServiceLoaded: tc.loaded,
ServiceWasRunning: tc.wasRunning,
snapshots: map[string]fileSnapshot{},
@@ -39,31 +51,68 @@ func TestRollbackDoesNotDisableAServiceItFoundRunning(t *testing.T) {
r.Rollback(Manifest{})
- if strings.Join(svc.calls, ",") != strings.Join(tc.want, ",") {
- t.Errorf("rollback called %v, want %v", svc.calls, tc.want)
+ if strings.Join(calls, ",") != strings.Join(tc.want, ",") {
+ t.Errorf("rollback called %v, want %v", calls, tc.want)
}
})
}
}
-// The restart has to happen after the files are restored. The collector reads its configuration at
-// startup, so restarting first would bring it up on the very config the failed install wrote.
-func TestRollbackRestartsAfterRestoringFiles(t *testing.T) {
+// The stop has to happen before the tracked files are restored, and this is the ordering the
+// supervised backend makes load-bearing: its "unit file" is its pidfile, so restoring it puts back
+// the pid from before the install -- a process already terminated. Unloading after that restore
+// looks up a dead pid, stops nothing, and leaves the collector this failed install started running
+// on the new config, holding the ports, with a second one started beside it.
+func TestRollbackStopsBeforeRestoringTheServiceStateFile(t *testing.T) {
dir := t.TempDir()
- path := dir + "/otelcol.yaml"
+ pidfile := filepath.Join(dir, "collector.pid")
+ if err := os.WriteFile(pidfile, []byte(`{"pid":100}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
- svc := &recordingService{}
+ var calls []string
r := &installRollback{
- Manager: svc,
+ Manager: recordingService{calls: &calls},
ServiceLoaded: true,
ServiceWasRunning: true,
snapshots: map[string]fileSnapshot{},
}
- r.Track(path)
+ // Track it the way Install does: the snapshot is taken while the file still holds the
+ // pre-install pid.
+ r.Track(pidfile)
+ if err := os.WriteFile(pidfile, []byte(`{"pid":200}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ calls = append(calls, "restore-boundary")
r.Rollback(Manifest{})
- if len(svc.calls) != 1 || svc.calls[0] != "restart" {
- t.Fatalf("service calls = %v, want a single restart", svc.calls)
+ unloadAt, boundaryAt := indexOf(calls, "unload"), indexOf(calls, "restore-boundary")
+ if unloadAt < 0 {
+ t.Fatalf("rollback never stopped the service: %v", calls)
+ }
+ if unloadAt < boundaryAt {
+ t.Fatalf("calls = %v: unload must come after the snapshot was taken", calls)
+ }
+ // The restored file must be the pre-install one, and the stop must already have happened by
+ // the time it lands.
+ data, err := os.ReadFile(pidfile)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(data), "100") {
+ t.Errorf("pidfile = %s, want the pre-install state restored", data)
+ }
+ if got := strings.Join(calls, ","); !strings.Contains(got, "unload,load") {
+ t.Errorf("calls = %v, want the service stopped and then brought back", calls)
+ }
+}
+
+func indexOf(haystack []string, needle string) int {
+ for i, v := range haystack {
+ if v == needle {
+ return i
+ }
}
+ return -1
}