Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

29.1 KB
Raw Blame
package agentupdate

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
)

var (
	recoveryELFBinary = append([]byte{0x7f, 'E', 'L', 'F'}, bytes.Repeat([]byte("x"), 128)...)
	recoveryPEBinary  = append([]byte{'M', 'Z'}, bytes.Repeat([]byte("x"), 130)...)
)

// recoveryUpdateClient answers the version check and binary download in
// memory, so CheckAndUpdate runs end to end without a socket.
func recoveryUpdateClient(version string, binary []byte, downloadStatus int, downloadStatusText string) *http.Client {
	return &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
		header := make(http.Header)
		status, statusText, body := http.StatusOK, "200 OK", []byte(nil)
		switch r.URL.Path {
		case "/api/agent/version":
			body = []byte(fmt.Sprintf(`{"version":%q}`, version))
		case "/download/pulse-agent":
			if downloadStatus != http.StatusOK {
				status, statusText = downloadStatus, downloadStatusText
				break
			}
			header.Set(checksumSHA256Header, checksum(binary))
			header.Set(signatureHeader, "signed-update")
			body = binary
		default:
			status, statusText = http.StatusNotFound, "404 Not Found"
		}
		return &http.Response{
			StatusCode:    status,
			Status:        statusText,
			Header:        header,
			Body:          io.NopCloser(bytes.NewReader(body)),
			ContentLength: int64(len(body)),
			Request:       r,
		}, nil
	})}
}

// stubRecoveryProcess fakes the process boundary: the executable path, the
// in-place restart (which returns restartErr) and the exit call.
func stubRecoveryProcess(t *testing.T, execPath string, restartErr error) *[]int {
	t.Helper()
	origExecutable, origEval, origRestart, origExit := osExecutableFn, evalSymlinksFn, restartProcessFn, exitFn
	t.Cleanup(func() {
		osExecutableFn, evalSymlinksFn, restartProcessFn, exitFn = origExecutable, origEval, origRestart, origExit
	})
	exits := &[]int{}
	osExecutableFn = func() (string, error) { return execPath, nil }
	evalSymlinksFn = func(path string) (string, error) { return path, nil }
	restartProcessFn = func(string) error { return restartErr }
	exitFn = func(code int) { *exits = append(*exits, code) }
	return exits
}

// isolateRecoveryPlatformProbes keeps a host's own Unraid or QNAP marker files
// from adding persistent-copy steps to a recorded replacement.
func isolateRecoveryPlatformProbes(t *testing.T) {
	t.Helper()
	missing := filepath.Join(t.TempDir(), "absent")
	origUnraid, origGetcfg, origQpkg := unraidVersionPath, qnapGetcfgPath, qnapQpkgConfPath
	t.Cleanup(func() {
		unraidVersionPath, qnapGetcfgPath, qnapQpkgConfPath = origUnraid, origGetcfg, origQpkg
	})
	unraidVersionPath, qnapGetcfgPath, qnapQpkgConfPath = missing, missing, missing
}

// useRecoveryPlatform selects which replacement path runs, whatever the host,
// and returns a payload whose magic that platform accepts.
func useRecoveryPlatform(t *testing.T, goos goOS) []byte {
	t.Helper()
	original := runtimeGOOS
	t.Cleanup(func() { runtimeGOOS = original })
	runtimeGOOS = goos
	if goos == goOSWindows {
		return recoveryPEBinary
	}
	return recoveryELFBinary
}

// newHelperRecoveryUpdater wires CheckAndUpdate to the fake typed helper with
// an in-place restart that fails after activation, as restartProcess reports it.
func newHelperRecoveryUpdater(t *testing.T, rollbackErr error) (*fakePrivilegedUpdate, *Updater, *[]int) {
	t.Helper()
	binary := useRecoveryPlatform(t, goOSLinux)
	helper := &fakePrivilegedUpdate{root: t.TempDir(), rollbackErr: rollbackErr}
	if err := securityutil.HardenPrivatePath(helper.root, 0o700); err != nil {
		t.Fatal(err)
	}
	exits := stubRecoveryProcess(t, "/usr/local/bin/pulse-agent", fmt.Errorf("failed to restart: %w", errors.New("exec refused")))
	u := New(Config{
		PulseURL:         "https://pulse.example.com",
		AgentName:        "pulse-agent",
		CurrentVersion:   "1.0.0",
		StateDir:         helper.root,
		PrivilegedUpdate: helper,
	})
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")
	u.selfTestFn = func(context.Context, string) error { return nil }
	return helper, u, exits
}

var helperRestartRollbackEvents = []string{"quarantine", "signature:signed-update", "stage", "activate", "cleanup", "rollback"}

// A helper-backed update whose in-place restart fails and is rolled back has
// restored the running binary. Base matched "failed to restart" in that error
// and called os.Exit(1), so the failure was never reported and the restarted
// agent began with an idle, error-free status.
func TestCheckAndUpdateReportsRolledBackHelperRestartFailureWithoutExiting(t *testing.T) {
	helper, u, exits := newHelperRecoveryUpdater(t, nil)

	u.CheckAndUpdate(context.Background())

	if len(*exits) != 0 {
		t.Fatalf("exit codes = %v, want none after the helper restored the running binary", *exits)
	}
	status := u.Snapshot()
	if status.State != UpdateStateError || status.LastSuccessAt != nil || !strings.Contains(status.LastError, "update rolled back") {
		t.Fatalf("status after rolled-back helper update = %+v", status)
	}
	if !reflect.DeepEqual(helper.events, helperRestartRollbackEvents) {
		t.Fatalf("helper events = %#v, want %#v", helper.events, helperRestartRollbackEvents)
	}
	if pending, err := LoadPendingPrivilegedUpdate(helper.root); err != nil || pending != nil {
		t.Fatalf("pending handoff after rollback = %#v, %v", pending, err)
	}
}

// A binary restored on disk is not a settled transaction. When clearing the
// local handoff fails after the helper rolled back, nothing on disk differs
// from the running binary, so the process keeps running and reports both
// facts; the stale handoff stays visible instead of being treated as settled.
func TestCheckAndUpdateReportsStaleHandoffWhenClearFailsAfterRollback(t *testing.T) {
	helper, u, exits := newHelperRecoveryUpdater(t, nil)
	origClear := clearPendingUpdateFn
	t.Cleanup(func() { clearPendingUpdateFn = origClear })
	clearPendingUpdateFn = func(string) error { return errors.New("fixture clear refused") }

	u.CheckAndUpdate(context.Background())

	if len(*exits) != 0 {
		t.Fatalf("exit codes = %v, want none: the binary on disk is the running one again", *exits)
	}
	status := u.Snapshot()
	if status.State != UpdateStateError || !strings.Contains(status.LastError, "update rolled back") ||
		!strings.Contains(status.LastError, "clear pending update handoff: fixture clear refused") {
		t.Fatalf("status after rollback with a failed clear = %+v", status)
	}
	if !reflect.DeepEqual(helper.events, helperRestartRollbackEvents) {
		t.Fatalf("helper events = %#v, want %#v", helper.events, helperRestartRollbackEvents)
	}
	pending, err := LoadPendingPrivilegedUpdate(helper.root)
	if err != nil || pending == nil || pending.Activation.ActivationID != helper.activation.ActivationID {
		t.Fatalf("stale handoff after failed clear = %#v, %v; want the rolled-back activation still recorded", pending, err)
	}
}

// When the helper cannot roll back, the activated binary stays on disk under a
// persisted pending handoff while this process runs the old one, so the exit
// that hands over to the service manager is kept.
func TestCheckAndUpdateExitsWhenHelperCannotRollBackRestartFailure(t *testing.T) {
	helper, u, exits := newHelperRecoveryUpdater(t, errors.New("helper unavailable"))

	u.CheckAndUpdate(context.Background())

	if !reflect.DeepEqual(*exits, []int{1}) {
		t.Fatalf("exit codes = %v, want [1]", *exits)
	}
	if status := u.Snapshot(); status.State != UpdateStateError || !strings.Contains(status.LastError, "typed helper rollback failed") {
		t.Fatalf("status after failed rollback = %+v", status)
	}
	pending, err := LoadPendingPrivilegedUpdate(helper.root)
	if err != nil || pending == nil || pending.Activation.ActivationID != helper.activation.ActivationID {
		t.Fatalf("pending handoff after failed rollback = %#v, %v", pending, err)
	}
}

// A download refused before any replacement leaves the running binary in
// place. Base exited anyway when the server's status text contained "failed
// to restart", dropping the error it had just recorded.
func TestCheckAndUpdateKeepsRunningWhenPreReplacementErrorMentionsRestart(t *testing.T) {
	isolateRecoveryPlatformProbes(t)
	binary := useRecoveryPlatform(t, goOSLinux)
	_, execPath := writeTempExec(t)
	exits := stubRecoveryProcess(t, execPath, nil)
	u := newUpdaterForTest("https://pulse.example.com")
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusNotFound, "404 failed to restart")

	u.CheckAndUpdate(context.Background())

	if len(*exits) != 0 {
		t.Fatalf("exit codes = %v, want none before any replacement", *exits)
	}
	if status := u.Snapshot(); status.State != UpdateStateError || !strings.Contains(status.LastError, "404 failed to restart") {
		t.Fatalf("status after refused download = %+v", status)
	}
	if got, err := os.ReadFile(execPath); err != nil || string(got) != "old-binary" {
		t.Fatalf("executable after refused download = %q, %v", got, err)
	}
}

type fsStep struct {
	op          string // create, close, chmod, syncfile, rename, remove, writefile, syncdir or restart
	path, to    string
	mode        os.FileMode
	servicePath bool // whether the service path existed after the step
}

type fsRecorder struct {
	servicePath string
	steps       []fsStep
	syncFileErr error
	syncDirErr  error
	restartErr  error
	failRename  func(from, to string) bool
}

func (r *fsRecorder) record(op, path, to string, mode os.FileMode) {
	_, err := os.Lstat(r.servicePath)
	r.steps = append(r.steps, fsStep{op: op, path: path, to: to, mode: mode, servicePath: err == nil})
}

func (r *fsRecorder) ops() []string {
	ops := make([]string, 0, len(r.steps))
	for _, step := range r.steps {
		ops = append(ops, step.op)
	}
	return ops
}

// recordUpdateFS routes the replacement's filesystem seams through a recorder.
// Creates, closes, chmods, renames, removals and writes really happen inside
// t.TempDir(); the durability barriers and the restart only record, or return
// the configured fixture error.
func recordUpdateFS(t *testing.T, servicePath string) *fsRecorder {
	t.Helper()
	rec := &fsRecorder{servicePath: servicePath}
	origCreate, origClose, origChmod, origRename := createTempFn, closeFileFn, chmodFn, renameFn
	origRemove, origWrite, origSyncFile, origSyncDir, origRestart := removeFileFn, writeFileFn, syncFileFn, syncDirFn, restartProcessFn
	t.Cleanup(func() {
		createTempFn, closeFileFn, chmodFn, renameFn = origCreate, origClose, origChmod, origRename
		removeFileFn, writeFileFn, syncFileFn, syncDirFn, restartProcessFn = origRemove, origWrite, origSyncFile, origSyncDir, origRestart
	})
	createTempFn = func(dir, pattern string) (*os.File, error) {
		file, err := origCreate(dir, pattern)
		if err == nil {
			rec.record("create", file.Name(), "", 0)
		}
		return file, err
	}
	closeFileFn = func(file *os.File) error {
		err := origClose(file)
		if err == nil {
			rec.record("close", file.Name(), "", 0)
		}
		return err
	}
	chmodFn = func(path string, mode os.FileMode) error {
		err := origChmod(path, mode)
		if err == nil {
			rec.record("chmod", path, "", mode)
		}
		return err
	}
	renameFn = func(from, to string) error {
		if rec.failRename != nil && rec.failRename(from, to) {
			return errors.New("fixture rename refused")
		}
		err := origRename(from, to)
		if err == nil {
			rec.record("rename", from, to, 0)
		}
		return err
	}
	removeFileFn = func(path string) error {
		err := origRemove(path)
		if err == nil {
			rec.record("remove", path, "", 0)
		}
		return err
	}
	writeFileFn = func(path string, data []byte, mode os.FileMode) error {
		err := origWrite(path, data, mode)
		if err == nil {
			rec.record("writefile", path, "", mode)
		}
		return err
	}
	syncFileFn = func(path string) error {
		if rec.syncFileErr != nil {
			return rec.syncFileErr
		}
		rec.record("syncfile", path, "", 0)
		return nil
	}
	syncDirFn = func(dir string) error {
		if rec.syncDirErr != nil {
			return rec.syncDirErr
		}
		rec.record("syncdir", dir, "", 0)
		return nil
	}
	restartProcessFn = func(string) error {
		rec.record("restart", "", "", 0)
		return rec.restartErr
	}
	return rec
}

type modelInode struct {
	complete, durableComplete bool
	mode, durableMode         os.FileMode
	updateInfo                bool
}

func describeServiceInode(inode *modelInode, durable bool) string {
	if inode == nil {
		return "missing"
	}
	complete, mode := inode.complete, inode.mode
	if durable {
		complete, mode = inode.durableComplete, inode.durableMode
	}
	if !complete {
		return "naming incomplete bytes"
	}
	if mode&0o111 == 0 {
		return fmt.Sprintf("not executable (%v)", mode)
	}
	return "valid"
}

// firstPowerLossViolation replays recorded steps against a deliberately
// pessimistic crash model. It is an assumption, not a measured filesystem: a
// file's bytes and mode reach disk only through a sync of that file after they
// were set, and each rename, removal or new file reaches disk independently,
// in any combination, unless a later sync of its directory forced it. After
// every step the visible service path, and in every combination the service
// path on disk, must name a complete executable; update evidence on disk must
// never outlive the replacement it describes. It returns the first violation.
func firstPowerLossViolation(steps []fsStep, servicePath string) string {
	type nsOp struct {
		step   fsStep
		inode  *modelInode
		forced bool
	}
	for crash := 0; crash <= len(steps); crash++ {
		previous := &modelInode{complete: true, durableComplete: true, mode: 0o755, durableMode: 0o755}
		visible := map[string]*modelInode{servicePath: previous}
		var ops []nsOp
		for _, step := range steps[:crash] {
			switch step.op {
			case "create":
				visible[step.path] = &modelInode{mode: 0o600}
				ops = append(ops, nsOp{step: step, inode: visible[step.path], forced: true})
			case "writefile":
				visible[step.path] = &modelInode{complete: true, mode: step.mode, updateInfo: true}
				ops = append(ops, nsOp{step: step, inode: visible[step.path]})
			case "close":
				if inode := visible[step.path]; inode != nil {
					inode.complete = true
				}
			case "chmod":
				if inode := visible[step.path]; inode != nil {
					inode.mode = step.mode
				}
			case "syncfile":
				if inode := visible[step.path]; inode != nil {
					inode.durableComplete, inode.durableMode = inode.complete, inode.mode
				}
			case "rename":
				visible[step.to] = visible[step.path]
				delete(visible, step.path)
				ops = append(ops, nsOp{step: step})
			case "remove":
				delete(visible, step.path)
				ops = append(ops, nsOp{step: step})
			case "syncdir":
				for i := range ops {
					if filepath.Dir(ops[i].step.path) == step.path {
						ops[i].forced = true
					}
				}
			}
		}
		if state := describeServiceInode(visible[servicePath], false); state != "valid" {
			return fmt.Sprintf("after step %d the visible service path is %s", crash, state)
		}
		var optional []int
		for i, op := range ops {
			if !op.forced {
				optional = append(optional, i)
			}
		}
		for mask := 0; mask < 1<<len(optional); mask++ {
			onDisk := make([]bool, len(ops))
			for i, op := range ops {
				onDisk[i] = op.forced
			}
			for bit, i := range optional {
				onDisk[i] = mask&(1<<bit) != 0
			}
			durable := map[string]*modelInode{servicePath: previous}
			var applied []string
			for i, op := range ops {
				if !onDisk[i] {
					continue
				}
				switch op.step.op {
				case "create", "writefile":
					durable[op.step.path] = op.inode
				case "rename":
					if inode, ok := durable[op.step.path]; ok {
						durable[op.step.to] = inode
						delete(durable, op.step.path)
					}
				case "remove":
					delete(durable, op.step.path)
				}
				if op.step.op != "create" {
					applied = append(applied, op.step.op+" "+filepath.Base(op.step.path))
				}
			}
			service := durable[servicePath]
			if state := describeServiceInode(service, true); state != "valid" {
				return fmt.Sprintf("power loss after step %d with %v on disk leaves the service path %s", crash, applied, state)
			}
			for _, inode := range durable {
				if inode.updateInfo && service == previous {
					return fmt.Sprintf("power loss after step %d with %v on disk keeps update evidence for a lost replacement", crash, applied)
				}
			}
		}
	}
	return ""
}

// Off Windows the staged binary is renamed over the service path only after
// its bytes and final executable mode were synced, the directory is synced
// before any cleanup or update evidence, and so after any step, or a power
// loss in the model above, the service path names a complete executable.
// Base moved the running binary aside first, leaving the path empty, and
// synced nothing.
func TestLocalReplacementKeepsExecutableServicePathThroughPowerLoss(t *testing.T) {
	isolateRecoveryPlatformProbes(t)
	binary := useRecoveryPlatform(t, goOSLinux)
	_, execPath := writeTempExec(t)
	stubRecoveryProcess(t, execPath, nil)
	rec := recordUpdateFS(t, execPath)
	u := newUpdaterForTest("https://pulse.example.com")
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")

	if err := u.performUpdateWithExecPath(context.Background(), execPath); err != nil {
		t.Fatalf("performUpdateWithExecPath: %v", err)
	}

	if violation := firstPowerLossViolation(rec.steps, execPath); violation != "" {
		t.Fatalf("%s; steps=%+v", violation, rec.steps)
	}
	for i, step := range rec.steps {
		if !step.servicePath {
			t.Fatalf("service path missing after step %d: %+v", i, step)
		}
	}
	want := []string{"create", "close", "chmod", "syncfile", "rename", "syncdir", "writefile", "restart"}
	if got := rec.ops(); !reflect.DeepEqual(got, want) {
		t.Fatalf("replacement steps = %v, want %v", got, want)
	}
	if got, err := os.ReadFile(execPath); err != nil || !bytes.Equal(got, binary) {
		t.Fatalf("executable after update = %q, %v", got, err)
	}
}

// A staged binary that cannot be made durable is a pre-replacement failure:
// the running binary stays untouched and nothing is renamed or restarted.
func TestLocalReplacementStagedSyncFailureKeepsRunningBinary(t *testing.T) {
	isolateRecoveryPlatformProbes(t)
	binary := useRecoveryPlatform(t, goOSLinux)
	dir, execPath := writeTempExec(t)
	stubRecoveryProcess(t, execPath, nil)
	rec := recordUpdateFS(t, execPath)
	rec.syncFileErr = errors.New("fixture fsync refused")
	u := newUpdaterForTest("https://pulse.example.com")
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")

	err := u.performUpdateWithExecPath(context.Background(), execPath)
	if err == nil || !strings.Contains(err.Error(), "failed to sync staged binary") {
		t.Fatalf("performUpdateWithExecPath error = %v", err)
	}
	if got := rec.ops(); !reflect.DeepEqual(got, []string{"create", "close", "chmod"}) {
		t.Fatalf("steps after sync failure = %v, want create, close, chmod only", got)
	}
	if got, readErr := os.ReadFile(execPath); readErr != nil || string(got) != "old-binary" {
		t.Fatalf("executable after sync failure = %q, %v", got, readErr)
	}
	if leftovers, _ := filepath.Glob(filepath.Join(dir, "pulse-agent-*.tmp")); len(leftovers) != 0 {
		t.Fatalf("staged temp files left behind: %v", leftovers)
	}
	if _, statErr := os.Stat(filepath.Join(dir, ".pulse-update-info")); !errors.Is(statErr, os.ErrNotExist) {
		t.Fatalf("update info written after a pre-replacement failure: %v", statErr)
	}
}

// Off Windows a refused rename leaves the service path exactly as it was: it
// is never moved aside, so there is nothing to restore. Base moved it aside
// first and depended on a second rename to put it back.
func TestLocalReplacementRenameFailureNeverEmptiesServicePath(t *testing.T) {
	isolateRecoveryPlatformProbes(t)
	binary := useRecoveryPlatform(t, goOSLinux)
	dir, execPath := writeTempExec(t)
	stubRecoveryProcess(t, execPath, nil)
	rec := recordUpdateFS(t, execPath)
	rec.failRename = func(from, to string) bool { return to == execPath && strings.HasSuffix(from, ".tmp") }
	u := newUpdaterForTest("https://pulse.example.com")
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")

	err := u.performUpdateWithExecPath(context.Background(), execPath)
	if err == nil || !strings.Contains(err.Error(), "failed to replace binary") {
		t.Fatalf("performUpdateWithExecPath error = %v", err)
	}
	for i, step := range rec.steps {
		if !step.servicePath || (step.op == "rename" && step.path == execPath) {
			t.Fatalf("service path moved or missing at step %d: %+v", i, step)
		}
	}
	if got := rec.ops(); !reflect.DeepEqual(got, []string{"create", "close", "chmod", "syncfile"}) {
		t.Fatalf("steps after refused rename = %v", got)
	}
	if got, readErr := os.ReadFile(execPath); readErr != nil || string(got) != "old-binary" {
		t.Fatalf("executable after refused rename = %q, %v", got, readErr)
	}
	if _, statErr := os.Stat(execPath + ".backup"); !errors.Is(statErr, os.ErrNotExist) {
		t.Fatalf("backup created although the service path was never moved: %v", statErr)
	}
	if leftovers, _ := filepath.Glob(filepath.Join(dir, "pulse-agent-*.tmp")); len(leftovers) != 0 {
		t.Fatalf("staged temp files left behind: %v", leftovers)
	}
}

// A failed directory sync leaves the replacement without a durability
// guarantee, so cleanup waits and any backup stays. The visible replacement is
// complete and the restart follows, so the "updated from" record is still
// written, which also replaces an earlier record nobody consumed. r2 skipped
// the write and left that earlier record to misattribute this update. No
// crash-durability claim is made on this path.
func TestLocalReplacementDirectorySyncFailureKeepsBackupAndReplacesStaleEvidence(t *testing.T) {
	isolateRecoveryPlatformProbes(t)
	binary := useRecoveryPlatform(t, goOSLinux)
	dir, execPath := writeTempExec(t)
	if err := os.WriteFile(execPath+".backup", []byte("stale-backup"), 0o755); err != nil {
		t.Fatal(err)
	}
	infoPath := filepath.Join(dir, ".pulse-update-info")
	if err := os.WriteFile(infoPath, []byte("0.9.0"), 0o644); err != nil {
		t.Fatal(err)
	}
	stubRecoveryProcess(t, execPath, nil)
	rec := recordUpdateFS(t, execPath)
	rec.syncDirErr = errors.New("fixture directory fsync refused")
	u := newUpdaterForTest("https://pulse.example.com")
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")

	if err := u.performUpdateWithExecPath(context.Background(), execPath); err != nil {
		t.Fatalf("performUpdateWithExecPath: %v", err)
	}
	if got := rec.ops(); !reflect.DeepEqual(got, []string{"create", "close", "chmod", "syncfile", "rename", "writefile", "restart"}) {
		t.Fatalf("steps with an unsynced directory = %v", got)
	}
	if got, err := os.ReadFile(execPath); err != nil || !bytes.Equal(got, binary) {
		t.Fatalf("executable after update = %q, %v", got, err)
	}
	if got, err := os.ReadFile(execPath + ".backup"); err != nil || string(got) != "stale-backup" {
		t.Fatalf("backup after unsynced directory = %q, %v", got, err)
	}
	if got, err := os.ReadFile(infoPath); err != nil || string(got) != "1.0.0" {
		t.Fatalf("update info = %q, %v; want this update's previous version, not the stale record", got, err)
	}
}

// Windows cannot rename over a running image, so its path still moves the
// running binary aside and the service path is empty until the second rename.
// This pins that platform gap so it is not mistaken for solved; the barriers
// around it are the same as elsewhere.
func TestWindowsReplacementPinsMissingServicePathGap(t *testing.T) {
	isolateRecoveryPlatformProbes(t)
	binary := useRecoveryPlatform(t, goOSWindows)
	_, execPath := writeTempExec(t)
	stubRecoveryProcess(t, execPath, nil)
	rec := recordUpdateFS(t, execPath)
	u := newUpdaterForTest("https://pulse.example.com")
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")

	if err := u.performUpdateWithExecPath(context.Background(), execPath); err != nil {
		t.Fatalf("performUpdateWithExecPath: %v", err)
	}
	want := []string{"create", "close", "chmod", "syncfile", "rename", "rename", "syncdir", "remove", "writefile", "restart"}
	if got := rec.ops(); !reflect.DeepEqual(got, want) {
		t.Fatalf("windows replacement steps = %v, want %v", got, want)
	}
	if aside := rec.steps[4]; aside.path != execPath || aside.servicePath {
		t.Fatalf("step 4 = %+v, want the running binary moved aside leaving the service path empty", aside)
	}
	if violation := firstPowerLossViolation(rec.steps, execPath); !strings.Contains(violation, "visible service path is missing") {
		t.Fatalf("power-loss model = %q, want the documented missing-path gap", violation)
	}
	if got, err := os.ReadFile(execPath); err != nil || !bytes.Equal(got, binary) {
		t.Fatalf("executable after update = %q, %v", got, err)
	}
}

// The Windows path keeps restoring the moved-aside binary when the second
// rename fails, as base did.
func TestWindowsReplacementRestoresBackupWhenSecondRenameFails(t *testing.T) {
	isolateRecoveryPlatformProbes(t)
	binary := useRecoveryPlatform(t, goOSWindows)
	_, execPath := writeTempExec(t)
	stubRecoveryProcess(t, execPath, nil)
	rec := recordUpdateFS(t, execPath)
	rec.failRename = func(from, to string) bool { return to == execPath && strings.HasSuffix(from, ".tmp") }
	u := newUpdaterForTest("https://pulse.example.com")
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")

	err := u.performUpdateWithExecPath(context.Background(), execPath)
	if err == nil || !strings.Contains(err.Error(), "failed to replace binary") {
		t.Fatalf("performUpdateWithExecPath error = %v", err)
	}
	var renames []string
	for _, step := range rec.steps {
		switch step.op {
		case "rename":
			renames = append(renames, filepath.Base(step.path)+"->"+filepath.Base(step.to))
		case "restart":
			t.Fatalf("restart after a failed replacement: %+v", rec.steps)
		}
	}
	base := filepath.Base(execPath)
	if want := []string{base + "->" + base + ".backup", base + ".backup->" + base}; !reflect.DeepEqual(renames, want) {
		t.Fatalf("renames = %v, want %v", renames, want)
	}
	if got, readErr := os.ReadFile(execPath); readErr != nil || string(got) != "old-binary" {
		t.Fatalf("executable after restore = %q, %v", got, readErr)
	}
	if _, statErr := os.Stat(execPath + ".backup"); !errors.Is(statErr, os.ErrNotExist) {
		t.Fatalf("backup left after restore: %v", statErr)
	}
}

// The local path replaces the executable before restarting, so a failed exec
// still exits for the service manager to start the new binary.
func TestCheckAndUpdateExitsWhenLocalReplacementCannotRestart(t *testing.T) {
	isolateRecoveryPlatformProbes(t)
	binary := useRecoveryPlatform(t, goOSLinux)
	_, execPath := writeTempExec(t)
	exits := stubRecoveryProcess(t, execPath, nil)
	rec := recordUpdateFS(t, execPath)
	rec.restartErr = fmt.Errorf("failed to restart: %w", errors.New("exec format error"))
	u := newUpdaterForTest("https://pulse.example.com")
	u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")

	u.CheckAndUpdate(context.Background())

	if !reflect.DeepEqual(*exits, []int{1}) {
		t.Fatalf("exit codes = %v, want [1]", *exits)
	}
	if ops := rec.ops(); len(ops) == 0 || ops[len(ops)-1] != "restart" {
		t.Fatalf("steps = %v, want the replacement to end in the restart attempt", ops)
	}
	if got, err := os.ReadFile(execPath); err != nil || !bytes.Equal(got, binary) {
		t.Fatalf("executable after local replacement = %q, %v", got, err)
	}
}

// The Unraid flash and QNAP persistent copies are what a reboot restores, so
// their bytes and final mode are synced before the rename and their directory
// after it; a copy that cannot be synced never replaces the previous one.
func TestPersistentBinaryCopySyncsFinalModeAroundRename(t *testing.T) {
	setup := func(t *testing.T) (string, string, *fsRecorder) {
		t.Helper()
		dir := t.TempDir()
		execPath := filepath.Join(dir, "running-agent")
		persistDir := filepath.Join(dir, "flash")
		if err := os.Mkdir(persistDir, 0o755); err != nil {
			t.Fatal(err)
		}
		persistPath := filepath.Join(persistDir, "pulse-agent")
		if err := os.WriteFile(execPath, []byte("new-binary"), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(persistPath, []byte("old-binary"), 0o755); err != nil {
			t.Fatal(err)
		}
		return execPath, persistPath, recordUpdateFS(t, persistPath)
	}

	t.Run("SyncedAroundRename", func(t *testing.T) {
		execPath, persistPath, rec := setup(t)
		New(Config{}).syncPersistentBinaryCopy(execPath, persistPath, "unraid")
		want := []string{"create", "close", "chmod", "syncfile", "rename", "syncdir"}
		if got := rec.ops(); !reflect.DeepEqual(got, want) {
			t.Fatalf("persistent copy steps = %v, want %v", got, want)
		}
		if rec.steps[2].path != rec.steps[3].path || rec.steps[2].mode != 0o755 {
			t.Fatalf("sync %+v does not follow the final chmod %+v of the same file", rec.steps[3], rec.steps[2])
		}
		if violation := firstPowerLossViolation(rec.steps, persistPath); violation != "" {
			t.Fatalf("%s; steps=%+v", violation, rec.steps)
		}
		if got, err := os.ReadFile(persistPath); err != nil || string(got) != "new-binary" {
			t.Fatalf("persistent copy = %q, %v", got, err)
		}
	})

	t.Run("SyncFailureKeepsPreviousCopy", func(t *testing.T) {
		execPath, persistPath, rec := setup(t)
		rec.syncFileErr = errors.New("fixture fsync refused")
		New(Config{}).syncPersistentBinaryCopy(execPath, persistPath, "unraid")
		if got := rec.ops(); !reflect.DeepEqual(got, []string{"create", "close", "chmod"}) {
			t.Fatalf("persistent copy steps after sync failure = %v", got)
		}
		if got, err := os.ReadFile(persistPath); err != nil || string(got) != "old-binary" {
			t.Fatalf("persistent copy after sync failure = %q, %v", got, err)
		}
		if leftovers, _ := filepath.Glob(persistPath + "-persist-*.tmp"); len(leftovers) != 0 {
			t.Fatalf("persistent temp files left behind: %v", leftovers)
		}
	})
}