Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(endpoint): clean rollback staging after successful restore

shukieshah6abd22563c47
diff --git a/cli/beacon/internal/endpoint/selfupdate/apply.go b/cli/beacon/internal/endpoint/selfupdate/apply.go
index 2faf26f6..e9de6bc6 100644
--- a/cli/beacon/internal/endpoint/selfupdate/apply.go
+++ b/cli/beacon/internal/endpoint/selfupdate/apply.go
@@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
+ "encoding/json"
"fmt"
"io"
"net/http"
@@ -188,6 +189,7 @@ func (a *Applier) Apply(ctx context.Context) (ApplyResult, error) {
message += "; rollback failed: " + rollbackErr.Error()
}
a.emit(false, result, message)
+ a.cleanupFailedUpdate(result, message, rollbackErr)
return result, fmt.Errorf("install update: %w", err)
}
@@ -199,6 +201,7 @@ func (a *Applier) Apply(ctx context.Context) (ApplyResult, error) {
message += "; rollback failed: " + rollbackErr.Error()
}
a.emit(false, result, message)
+ a.cleanupFailedUpdate(result, message, rollbackErr)
return result, fmt.Errorf("post-install collector restart failed: %w", err)
}
}
@@ -210,6 +213,7 @@ func (a *Applier) Apply(ctx context.Context) (ApplyResult, error) {
message += "; rollback failed: " + rollbackErr.Error()
}
a.emit(false, result, message)
+ a.cleanupFailedUpdate(result, message, rollbackErr)
return result, fmt.Errorf("post-install health check failed: %w", err)
}
@@ -235,6 +239,11 @@ func (a *Applier) stageDir() string {
}
func (a *Applier) cleanupSuccessfulUpdate() {
+ a.cleanupLargeUpdateArtifacts()
+ _ = os.Remove(filepath.Join(a.stageDir(), "last_failure.json"))
+}
+
+func (a *Applier) cleanupLargeUpdateArtifacts() {
stage := a.stageDir()
for _, pattern := range []string{
filepath.Join(stage, "download*"),
@@ -250,6 +259,41 @@ func (a *Applier) cleanupSuccessfulUpdate() {
}
}
+type failureDiagnostic struct {
+ Timestamp string `json:"timestamp"`
+ FromVersion string `json:"from_version,omitempty"`
+ ToVersion string `json:"to_version,omitempty"`
+ Reason string `json:"reason"`
+ RolledBack bool `json:"rolled_back"`
+ RollbackError string `json:"rollback_error,omitempty"`
+}
+
+func (a *Applier) cleanupFailedUpdate(result ApplyResult, reason string, rollbackErr error) {
+ a.writeFailureDiagnostic(result, reason, rollbackErr)
+ if rollbackErr != nil || !result.RolledBack {
+ return
+ }
+ a.cleanupLargeUpdateArtifacts()
+}
+
+func (a *Applier) writeFailureDiagnostic(result ApplyResult, reason string, rollbackErr error) {
+ diag := failureDiagnostic{
+ Timestamp: a.clock()().UTC().Format(time.RFC3339),
+ FromVersion: result.FromVersion,
+ ToVersion: result.ToVersion,
+ Reason: reason,
+ RolledBack: result.RolledBack,
+ }
+ if rollbackErr != nil {
+ diag.RollbackError = rollbackErr.Error()
+ }
+ data, err := json.MarshalIndent(diag, "", " ")
+ if err != nil {
+ return
+ }
+ _ = os.WriteFile(filepath.Join(a.stageDir(), "last_failure.json"), append(data, '\n'), 0o644)
+}
+
func (a *Applier) prefix() string {
if a.InstallPrefix != "" {
return a.InstallPrefix
diff --git a/cli/beacon/internal/endpoint/selfupdate/apply_test.go b/cli/beacon/internal/endpoint/selfupdate/apply_test.go
index bc2225d4..d742509d 100644
--- a/cli/beacon/internal/endpoint/selfupdate/apply_test.go
+++ b/cli/beacon/internal/endpoint/selfupdate/apply_test.go
@@ -119,6 +119,10 @@ func TestApplyCleansSuccessfulUpdateStaging(t *testing.T) {
if err := os.WriteFile(staleRollback, []byte("old"), 0o755); err != nil {
t.Fatal(err)
}
+ staleFailure := filepath.Join(a.StageDir, "last_failure.json")
+ if err := os.WriteFile(staleFailure, []byte(`{"reason":"old failure"}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
res, err := a.Apply(context.Background())
if err != nil {
@@ -136,6 +140,9 @@ func TestApplyCleansSuccessfulUpdateStaging(t *testing.T) {
t.Fatalf("%s still exists or unexpected error: %v", path, err)
}
}
+ if _, err := os.Stat(staleFailure); !os.IsNotExist(err) {
+ t.Fatalf("stale failure diagnostic still exists or unexpected error: %v", err)
+ }
if _, err := os.Stat(filepath.Join(a.StageDir, ".update.lock")); err != nil {
t.Fatalf("lock file should remain reusable: %v", err)
}
@@ -231,6 +238,20 @@ func TestApplyHealthFailRollsBack(t *testing.T) {
if !strings.Contains(string(got), "OLD-BINARY") {
t.Fatalf("rollback did not restore old binary, got %q", got)
}
+ if _, err := os.Stat(filepath.Join(a.StageDir, "rollback")); !os.IsNotExist(err) {
+ t.Fatalf("successful rollback should clean rollback snapshot, stat err=%v", err)
+ }
+ diag := string(mustRead(t, filepath.Join(a.StageDir, "last_failure.json")))
+ for _, want := range []string{
+ `"from_version": "0.0.1"`,
+ `"to_version": "9.9.9"`,
+ `"rolled_back": true`,
+ "post-install health check failed",
+ } {
+ if !strings.Contains(diag, want) {
+ t.Fatalf("failure diagnostic missing %q:\n%s", want, diag)
+ }
+ }
}
func TestApplyGatekeeperAbortsBeforeInstall(t *testing.T) {
@@ -349,6 +370,13 @@ func TestApplyRestartsCollectorBeforeHealthCheck(t *testing.T) {
if !strings.Contains(string(mustRead(t, a.LogPath)), "rollback failed") {
t.Fatalf("expected rollback restart failure telemetry, got %s", mustRead(t, a.LogPath))
}
+ if _, err := os.Stat(filepath.Join(a.StageDir, "rollback")); err != nil {
+ t.Fatalf("failed rollback should preserve rollback materials: %v", err)
+ }
+ diag := string(mustRead(t, filepath.Join(a.StageDir, "last_failure.json")))
+ if !strings.Contains(diag, "rollback_error") || !strings.Contains(diag, "launchctl failed") {
+ t.Fatalf("failure diagnostic should preserve rollback error, got %s", diag)
+ }
}
func TestSkipGatekeeperRequiresInsecureFlag(t *testing.T) {