Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

feat(rexctl): add upgrade command

Hampusc8b59edd8a04
diff --git a/fluxer_rexctl/cmd/upgrade.go b/fluxer_rexctl/cmd/upgrade.go
new file mode 100644
index 000000000..55a3049f0
--- /dev/null
+++ b/fluxer_rexctl/cmd/upgrade.go
@@ -0,0 +1,1107 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "maps"
+ "os"
+ "path/filepath"
+ "runtime"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/backup"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/checks"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/deploy"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/engine"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/host"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/instance"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/release"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/secrets"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/settings"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/setup"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/ui"
+)
+
+const (
+ readyTimeout = 10 * time.Minute
+ recoveryTimeout = 20 * time.Minute
+ scratchDir = "render"
+)
+
+var fatalChecks = map[string]bool{"services": true, "health": true}
+
+type upgradeFlags struct {
+ Check bool
+ To string
+ Prefetch bool
+ NoBackup bool
+ DryRun bool
+ Resume string
+}
+
+type upgradeCheckJSON struct {
+ Current string `json:"current"`
+ Latest string `json:"latest"`
+ Available bool `json:"available"`
+ MinFrom string `json:"min_from"`
+ SchemaChanged bool `json:"schema_changed"`
+ Blocked bool `json:"blocked"`
+}
+
+func newUpgradeCmd(app *App) *cobra.Command {
+ var f upgradeFlags
+ c := &cobra.Command{
+ Use: "upgrade",
+ Short: "Upgrade rexctl and Fluxer to the latest release",
+ Args: noArgs,
+ RunE: func(c *cobra.Command, _ []string) error {
+ return runUpgrade(c.Context(), app, &f)
+ },
+ }
+ fl := c.Flags()
+ fl.BoolVar(&f.Check, "check", false, "show whether an upgrade is available without changing anything")
+ fl.StringVar(&f.To, "to", "", "upgrade to this version, not the latest")
+ fl.BoolVar(&f.Prefetch, "prefetch", false, "download the new version and upgrade later")
+ fl.BoolVar(&f.NoBackup, "no-backup", false, "skip the backup before upgrading")
+ fl.BoolVar(&f.DryRun, "dry-run", false, "show what would happen without changing anything")
+ fl.StringVar(&f.Resume, release.ResumeFlag, "", "")
+ _ = fl.MarkHidden(release.ResumeFlag)
+ app.jsonFlag(c)
+ return c
+}
+
+func runUpgrade(ctx context.Context, app *App, f *upgradeFlags) error {
+ if f.Check {
+ if f.Prefetch || f.NoBackup || f.DryRun {
+ return ui.Usage("Use --check on its own, or with --to and --json.")
+ }
+ return runUpgradeCheck(ctx, app, f)
+ }
+ if app.Flags.JSON {
+ return ui.Usage("Use --json together with --check.")
+ }
+ if f.Prefetch && f.NoBackup {
+ return ui.Usage("Use --no-backup when you upgrade, not with --prefetch.")
+ }
+ if f.Resume != "" {
+ return resumeUpgrade(ctx, app, f)
+ }
+ inst, tookOver, err := upgradeInstance(ctx, app, f.DryRun)
+ if err != nil || inst == nil {
+ return err
+ }
+ if err := managed(inst); err != nil {
+ return err
+ }
+ var e *engine.Engine
+ if !f.DryRun {
+ if e, err = app.Engine(ctx, inst); err != nil {
+ return err
+ }
+ if inst.State.Upgrade != nil {
+ if err := app.Locked(ctx, inst, func() error { return recoverUpgrade(ctx, app, inst, true) }); err != nil {
+ return err
+ }
+ }
+ }
+ cur := inst.State.Release
+ if f.DryRun && inst.State.Upgrade != nil {
+ cur = printRecoveryPlan(app, inst)
+ }
+ rel, err := resolveRelease(ctx, f.To)
+ if err != nil {
+ return err
+ }
+ if err := release.CheckPath(cur, rel); err != nil {
+ return tooOldError(err)
+ }
+ from, to := versionPair(cur, rel.Version)
+ switch cmp := release.Compare(rel.Version, cur); {
+ case cmp == 0:
+ app.UI.OK("Fluxer %s is up to date.", to)
+ return nil
+ case cmp < 0:
+ return ui.Refused(fmt.Sprintf("Fluxer %s is older than the release you run, %s.", to, from), "Run rexctl rollback to go back to the release you ran before.")
+ }
+ if e != nil {
+ if err := release.CheckTarget(e, rel); err != nil {
+ return floorRefusal(err, to)
+ }
+ }
+ if release.Valid(cur) {
+ app.UI.Info("Fluxer %s is available. You're on %s.", to, from)
+ } else {
+ app.UI.Info("Fluxer %s is available.", to)
+ }
+ needSelf := release.Compare(rel.Version, app.Version) != 0
+ if f.DryRun {
+ for _, k := range keptCopies(inst) {
+ warnKeptCopy(app.UI, k.Name, k.Copy, true)
+ }
+ printUpgradePlan(app, f, needSelf, to)
+ return nil
+ }
+ if f.Prefetch {
+ app.UI.Info("Fluxer keeps running while the new version downloads.")
+ } else {
+ if ok, err := confirmKeptCopies(app, inst); err != nil || !ok {
+ if err == nil {
+ app.UI.Info("Nothing changed.")
+ }
+ return err
+ }
+ app.UI.Info("Fluxer keeps running while the new version downloads, then stops for about 2 minutes.")
+ ok, err := app.UI.Confirm("Upgrade?", true)
+ if err != nil {
+ return err
+ }
+ if !ok {
+ if tookOver {
+ app.UI.Info("rexctl took over your instance. Run rexctl upgrade when you're ready.")
+ } else {
+ app.UI.Info("Nothing changed.")
+ }
+ return nil
+ }
+ }
+ if !needSelf {
+ return app.Change(ctx, inst, func(e *engine.Engine) error {
+ if err := setPhase(inst, cur, rel.Version, release.PhaseResolved); err != nil {
+ return err
+ }
+ return upgradeStack(ctx, app, inst, e, f)
+ })
+ }
+ asset, err := rel.AssetFor(runtime.GOOS, runtime.GOARCH)
+ if err != nil {
+ return ui.Failed(fmt.Sprintf("There's no rexctl %s for this machine.", to), "")
+ }
+ err = app.Change(ctx, inst, func(*engine.Engine) error {
+ if err := setPhase(inst, cur, rel.Version, release.PhaseResolved); err != nil {
+ return err
+ }
+ err := app.UI.Step(ctx, "Updating rexctl", func(ctx context.Context, s *ui.Step) error {
+ if release.Valid(app.Version) && release.Compare(rel.Version, app.Version) < 0 {
+ s.Done("Switched rexctl back to %s.", to)
+ } else {
+ s.Done("Updated rexctl to %s.", to)
+ }
+ return release.SelfUpdate(ctx, asset)
+ })
+ if err != nil {
+ clearUpgrade(inst)
+ return selfUpdateError(err, inst)
+ }
+ return setPhase(inst, cur, rel.Version, release.PhaseSelfUpdated)
+ })
+ if err != nil {
+ return err
+ }
+ if err := release.Reexec(resumeArgs(os.Args[1:], release.PhaseSelfUpdated)); err != nil {
+ return ui.Wrap(err, "Updated rexctl but couldn't start it again.", "Run rexctl upgrade again.")
+ }
+ return nil
+}
+
+func floorRefusal(err error, to string) error {
+ var old *release.EngineTooOldError
+ if errors.As(err, &old) {
+ need, have := strings.TrimSuffix(old.Need, ".0"), old.Have
+ msg := fmt.Sprintf("Fluxer keeps running. This release needs %s %s or newer and this machine has %s.", old.Kind.Name(), need, have)
+ if old.Kind == engine.Podman {
+ return &ui.ExitError{Code: ui.ExitRefused, Msg: msg, Fix: "Upgrade Podman, or move to Docker with rexctl backup and then rexctl restore <file> --engine docker.", Err: err}
+ }
+ return &ui.ExitError{Code: ui.ExitRefused, Msg: msg, Fix: "Upgrade Docker, then run rexctl upgrade again.", Err: err}
+ }
+ var schema *release.OlderSchemaError
+ if errors.As(err, &schema) {
+ return &ui.ExitError{Code: ui.ExitRefused, Msg: fmt.Sprintf("Fluxer %s can't read this instance's settings, which a newer rexctl saved.", to), Fix: "Run rexctl upgrade without --to to stay on a newer release.", Err: err}
+ }
+ return err
+}
+
+func runUpgradeCheck(ctx context.Context, app *App, f *upgradeFlags) error {
+ dir, err := app.Dir()
+ if err != nil {
+ return err
+ }
+ var cur string
+ inst, err := instance.Find(dir)
+ switch {
+ case errors.Is(err, instance.ErrLegacy):
+ if legacy, lerr := instance.DetectLegacy(dir); lerr == nil {
+ cur = legacy.ImageTag
+ }
+ case err != nil:
+ if _, ierr := app.Instance(); ierr != nil {
+ return ierr
+ }
+ return err
+ default:
+ cur = inst.State.Release
+ }
+ rel, err := resolveRelease(ctx, f.To)
+ if err != nil {
+ return err
+ }
+ out := upgradeCheckJSON{
+ Current: cur,
+ Latest: rel.Version,
+ Available: release.Compare(rel.Version, cur) > 0,
+ MinFrom: rel.MinFrom,
+ SchemaChanged: rel.SchemaChanged,
+ }
+ pathErr := release.CheckPath(cur, rel)
+ out.Blocked = pathErr != nil
+ if app.Flags.JSON {
+ return app.UI.JSON(out)
+ }
+ from, to := versionPair(cur, rel.Version)
+ if !out.Available {
+ app.UI.OK("Fluxer %s is up to date.", from)
+ return nil
+ }
+ if release.Valid(cur) {
+ app.UI.Info("Fluxer %s is available. You're on %s.", to, from)
+ } else {
+ app.UI.Info("Fluxer %s is available.", to)
+ }
+ if inst != nil && inst.State.FailedRelease == rel.Version {
+ fix := "Run rexctl logs api to see why before you try again."
+ if log := inst.Path(instance.StateDirName, "upgrade-"+release.DirName(rel.Version)+".log"); fileExists(log) {
+ fix = fmt.Sprintf("Read %s to see why before you try again.", log)
+ }
+ app.UI.Warn(fmt.Sprintf("The last upgrade to Fluxer %s didn't start.", to), fix)
+ return nil
+ }
+ var tooOld *release.TooOldError
+ if errors.As(pathErr, &tooOld) {
+ app.UI.Warn(tooOld.Error(), fmt.Sprintf("Run %s first, then run rexctl upgrade again.", tooOld.Command()))
+ return nil
+ }
+ for _, k := range keptCopies(inst) {
+ warnKeptCopy(app.UI, k.Name, k.Copy, true)
+ }
+ app.UI.Info("Run rexctl upgrade when you're ready.")
+ return nil
+}
+
+func upgradeInstance(ctx context.Context, app *App, dryRun bool) (*instance.Instance, bool, error) {
+ dir, err := app.Dir()
+ if err != nil {
+ return nil, false, err
+ }
+ inst, err := instance.Find(dir)
+ if errors.Is(err, instance.ErrLegacy) {
+ legacy, err := instance.DetectLegacy(dir)
+ if err != nil {
+ return nil, false, err
+ }
+ inst, kept, err := runTakeover(ctx, app, legacy, takeoverOptions{DryRun: dryRun, Then: "Then rexctl upgrades it to the latest Fluxer release."})
+ if err != nil || inst == nil {
+ return nil, false, err
+ }
+ if len(kept) > 0 {
+ app.UI.Info("rexctl took over your instance and didn't upgrade it. Your changes keep working until you run rexctl upgrade.")
+ return nil, false, nil
+ }
+ return inst, true, nil
+ }
+ if err != nil {
+ inst, err := app.Instance()
+ return inst, false, err
+ }
+ return inst, false, nil
+}
+
+func resumeUpgrade(ctx context.Context, app *App, f *upgradeFlags) error {
+ inst, err := app.Instance()
+ if err != nil {
+ return err
+ }
+ return app.Change(ctx, inst, func(e *engine.Engine) error {
+ p := inst.State.Upgrade
+ if p == nil || p.Phase != f.Resume {
+ return ui.Refused("There's no upgrade to continue.", "Run rexctl upgrade.")
+ }
+ if release.Valid(app.Version) && release.Compare(app.Version, p.To) != 0 {
+ clearUpgrade(inst)
+ return ui.Failed(fmt.Sprintf("rexctl didn't update to %s.", p.To), "Run rexctl upgrade again.")
+ }
+ if err := refreshBootUnit(ctx, inst, e, nil); err != nil {
+ app.UI.Detail("%v", err)
+ }
+ return upgradeStack(ctx, app, inst, e, f)
+ })
+}
+
+func printUpgradePlan(app *App, f *upgradeFlags, needSelf bool, to string) {
+ if needSelf {
+ app.UI.Info("rexctl updates itself to %s.", to)
+ }
+ app.UI.Info("Fluxer %s downloads while your instance keeps running.", to)
+ if f.Prefetch {
+ app.UI.Info("Dry run. Nothing changed.")
+ return
+ }
+ if f.NoBackup {
+ app.UI.Info("Your instance isn't backed up first.")
+ } else {
+ app.UI.Info("Your instance is backed up first.")
+ }
+ app.UI.Info("Fluxer stops for about 2 minutes while it moves to %s.", to)
+ app.UI.Info("If %s doesn't start, rexctl goes back to the release you have today.", to)
+ app.UI.Info("Dry run. Nothing changed.")
+}
+
+func upgradeStack(ctx context.Context, app *App, inst *instance.Instance, e *engine.Engine, f *upgradeFlags) error {
+ pend := inst.State.Upgrade
+ from, to := pend.From, pend.To
+ fromShort, toShort := versionPair(from, to)
+ set := stack.Embedded()
+ images, err := stack.EmbeddedImages()
+ if err != nil {
+ clearUpgrade(inst)
+ return ui.Wrap(err, "This rexctl has no Fluxer release built in.", "Download rexctl again, then run rexctl upgrade.")
+ }
+ if err := release.Snapshot(inst, from, refList(inst.State.Images.Refs)); err != nil {
+ clearUpgrade(inst)
+ return ui.Wrap(err, fmt.Sprintf("Couldn't save the files of %s.", fluxerName(fromShort)), "Check that the disk isn't full, then run rexctl upgrade again.")
+ }
+ if err := release.WriteRelease(inst, set); err != nil {
+ clearUpgrade(inst)
+ return ui.Wrap(err, fmt.Sprintf("Couldn't save the files of Fluxer %s.", toShort), "Check that the disk isn't full, then run rexctl upgrade again.")
+ }
+ next, pins, err := nextState(inst, set, images, to)
+ if err != nil {
+ clearUpgrade(inst)
+ return ui.Wrap(err, fmt.Sprintf("Couldn't update your settings for Fluxer %s.", toShort), "Run rexctl upgrade --verbose to see why.")
+ }
+ if err := release.CheckPostgres(postgresRef(inst.State), postgresRef(next)); err != nil {
+ clearUpgrade(inst)
+ var pg *release.PostgresMajorError
+ if errors.As(err, &pg) {
+ return ui.Refused(fmt.Sprintf("Fluxer %s moves your database to a new major version, and rexctl can't do that yet.", toShort), fmt.Sprintf("Nothing changed. You're still on %s.", fluxerName(fromShort)))
+ }
+ return ui.Wrap(err, fmt.Sprintf("Couldn't read the database version for Fluxer %s.", toShort), "Run rexctl upgrade --verbose to see why.")
+ }
+ if err := release.Snapshot(&instance.Instance{Dir: inst.Dir, State: next}, to, refList(next.Images.Refs)); err != nil {
+ clearUpgrade(inst)
+ return ui.Wrap(err, fmt.Sprintf("Couldn't save the settings for Fluxer %s.", toShort), "Check that the disk isn't full, then run rexctl upgrade again.")
+ }
+ old := e.Named(inst.Project(), inst.Dir)
+ if c, ok := checks.Get("disk"); ok && c.Run != nil {
+ if r := checks.RunIn(ctx, checkEnv(ctx, inst, e, old), checks.Installed, c); r.Status == checks.Fail || r.Status == checks.Warn {
+ clearUpgrade(inst)
+ return ui.Refused(r.Message, r.Fix)
+ }
+ }
+ if err := pullRelease(ctx, app, inst, e, set, next, toShort); err != nil {
+ clearUpgrade(inst)
+ if ctx.Err() != nil {
+ app.UI.Info("Stopped the upgrade. %s is still running.", capitalise(fluxerName(fromShort)))
+ return ctx.Err()
+ }
+ return err
+ }
+ if err := setPhase(inst, from, to, release.PhasePulled); err != nil {
+ return err
+ }
+ if f.Prefetch {
+ clearUpgrade(inst)
+ app.UI.Info("Run rexctl upgrade when you're ready. It takes about 2 minutes.")
+ return nil
+ }
+ if f.NoBackup {
+ app.UI.Warn("Skipped the backup.", "")
+ } else {
+ err := app.UI.Step(ctx, "Backing up your instance", func(ctx context.Context, s *ui.Step) error {
+ info, err := backup.Create(ctx, inst, old, backup.CreateOptions{Reason: "upgrade", OnEvent: app.progress(s, "Starting"), Progress: func(stage string) {
+ if t := backupStageTitle(stage); t != "" {
+ s.Title(t)
+ }
+ }})
+ if errors.Is(err, backup.ErrNeverStarted) {
+ s.Done("There was nothing to back up yet.")
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ inst.State.Upgrade.Backup = info.Path
+ s.Done("Backed up your instance.")
+ return nil
+ })
+ if err != nil {
+ fix := "Run rexctl backup to see why, then run rexctl upgrade again."
+ if logPath := writeBackupLog(inst, "upgrade-"+release.DirName(to)+".log", err); logPath != "" {
+ fix = fmt.Sprintf("Read %s to see why, then run rexctl upgrade again.", logPath)
+ }
+ report := ui.Wrap(err, fmt.Sprintf("Couldn't back up your instance. %s is still running.", capitalise(fluxerName(fromShort))), fix)
+ var be *ui.ExitError
+ if errors.As(backupCreateErr(err), &be) && be.Err == nil {
+ report.Msg = be.Msg + " " + capitalise(fluxerName(fromShort)) + " is still running."
+ report.Fix = be.Fix
+ }
+ return abortUpgrade(ctx, app, inst, e, err, report)
+ }
+ if err := setPhase(inst, from, to, release.PhaseBackedUp); err != nil {
+ return err
+ }
+ }
+ if err := setPhase(inst, from, to, release.PhaseMigrated); err != nil {
+ return abortUpgrade(ctx, app, inst, e, err, ui.Wrap(err, "Couldn't save your settings.", "Run rexctl upgrade --verbose to see why."))
+ }
+ r, err := stack.Render(set, stack.OverlaysFor(next.Settings), next.Images.Refs, inst.Project(), inst.Dir)
+ if err != nil {
+ return abortUpgrade(ctx, app, inst, e, err, ui.Wrap(err, fmt.Sprintf("Couldn't write the files for Fluxer %s.", toShort), "Run rexctl upgrade --verbose to see why."))
+ }
+ inst.State.Rendered = r.Hashes
+ next.Rendered = r.Hashes
+ if err := setPhase(inst, from, to, release.PhaseRecreated); err != nil {
+ return abortUpgrade(ctx, app, inst, e, err, ui.Wrap(err, "Couldn't save your settings.", "Run rexctl upgrade --verbose to see why."))
+ }
+ target := &instance.Instance{Dir: inst.Dir, State: next}
+ p, err := deploy.Load(ctx, target, e)
+ if err != nil {
+ return abortUpgrade(ctx, app, inst, e, err, engineFailure(err))
+ }
+ waiting, err := startRelease(ctx, app, target, e, p, toShort)
+ if err != nil {
+ if ctx.Err() == nil {
+ return failedStart(ctx, app, inst, e, p, waiting, err)
+ }
+ return abortUpgrade(ctx, app, inst, e, err, ctx.Err())
+ }
+ if err := setPhase(inst, from, to, release.PhaseVerified); err != nil {
+ return err
+ }
+ for _, pin := range pins {
+ app.UI.Info("rexctl kept your own %s image.", pin)
+ }
+ return finishUpgrade(ctx, app, inst, e)
+}
+
+func startRelease(ctx context.Context, app *App, target *instance.Instance, e *engine.Engine, p *engine.Project, toShort string) ([]string, error) {
+ var waiting []string
+ err := app.UI.Step(ctx, "Starting Fluxer "+toShort, func(ctx context.Context, s *ui.Step) error {
+ if err := p.Pull(ctx, app.progress(s, "Downloading")); err != nil {
+ return err
+ }
+ if err := upAll(ctx, p, app.progress(s, "Starting")); err != nil {
+ return err
+ }
+ show := showReadiness(s, "Starting Fluxer "+toShort)
+ err := p.WaitReady(ctx, readyTimeout, func(r engine.Readiness) {
+ waiting = append(append([]string(nil), r.NotReady...), r.Failed...)
+ show(r)
+ })
+ if err != nil {
+ return err
+ }
+ waiting = nil
+ for _, r := range checks.Run(ctx, checkEnv(ctx, target, e, p), checks.Installed) {
+ if r.Status != checks.Fail {
+ continue
+ }
+ if fatalChecks[r.ID] {
+ return errors.New(r.Message)
+ }
+ app.UI.Warn(r.Message, r.Fix)
+ }
+ s.Done("Started Fluxer %s.", toShort)
+ return nil
+ })
+ return waiting, err
+}
+
+func retryUpgrade(ctx context.Context, app *App, inst *instance.Instance, e *engine.Engine) error {
+ pend := inst.State.Upgrade
+ _, toShort := versionPair(pend.From, pend.To)
+ next, err := release.SnapshotState(inst, pend.To)
+ if err != nil {
+ return ui.Wrap(err, fmt.Sprintf("Couldn't read the settings for Fluxer %s.", toShort), "Run rexctl doctor to see what's wrong.")
+ }
+ set, err := release.LoadSet(inst, pend.To)
+ if err != nil {
+ return ui.Wrap(err, fmt.Sprintf("Couldn't read the files for Fluxer %s.", toShort), "Run rexctl doctor to see what's wrong.")
+ }
+ r, err := stack.Render(set, stack.OverlaysFor(next.Settings), next.Images.Refs, inst.Project(), inst.Dir)
+ if err != nil {
+ return ui.Wrap(err, fmt.Sprintf("Couldn't write the files for Fluxer %s.", toShort), "Run rexctl upgrade --verbose to see why.")
+ }
+ inst.State.Rendered = r.Hashes
+ next.Rendered = r.Hashes
+ if err := inst.Save(); err != nil {
+ return ui.Wrap(err, "Couldn't save your instance's state.", "Run rexctl doctor to see what's wrong.")
+ }
+ if err := e.Named(inst.Project(), inst.Dir).RemoveReplacements(ctx); err != nil {
+ return engineFailure(err)
+ }
+ target := &instance.Instance{Dir: inst.Dir, State: next}
+ p, err := deploy.Load(ctx, target, e)
+ if err != nil {
+ return engineFailure(err)
+ }
+ waiting, err := startRelease(ctx, app, target, e, p, toShort)
+ if err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ fix := release.UnfinishedFix(pend)
+ if logPath := writeFailureLog(ctx, inst, p, "upgrade-"+release.DirName(pend.To)+".log", err, waiting); logPath != "" {
+ fix = fmt.Sprintf("Read %s to see why. %s", logPath, fix)
+ }
+ return ui.Wrap(err, fmt.Sprintf("Fluxer %s didn't start either.", toShort), fix)
+ }
+ if err := setPhase(inst, pend.From, pend.To, release.PhaseVerified); err != nil {
+ return err
+ }
+ return finishUpgrade(ctx, app, inst, e)
+}
+
+func forwardable(inst *instance.Instance) bool {
+ pend := inst.State.Upgrade
+ return pend != nil && pend.BackFailed && phaseIndex(pend.Phase) >= phaseIndex(release.PhaseRecreated) && release.HasSnapshot(inst, pend.To)
+}
+
+func printRecoveryPlan(app *App, inst *instance.Instance) string {
+ pend := inst.State.Upgrade
+ fromShort, toShort := versionPair(pend.From, pend.To)
+ phase := phaseIndex(pend.Phase)
+ switch {
+ case phase >= phaseIndex(release.PhaseVerified):
+ app.UI.Info("The upgrade to Fluxer %s didn't finish. rexctl finishes it first.", toShort)
+ return pend.To
+ case forwardable(inst):
+ app.UI.Info("The upgrade to Fluxer %s didn't finish. rexctl starts Fluxer %s again.", toShort, toShort)
+ return pend.To
+ case phase >= phaseIndex(release.PhasePulled) && release.HasSnapshot(inst, pend.From):
+ app.UI.Info("The upgrade to Fluxer %s didn't finish. rexctl goes back to %s first.", toShort, fluxerName(fromShort))
+ if phase >= phaseIndex(release.PhaseRecreated) && release.HasSnapshot(inst, pend.To) {
+ app.UI.Info("If %s doesn't start, rexctl starts Fluxer %s again.", fluxerName(fromShort), toShort)
+ }
+ }
+ return pend.From
+}
+
+func nextState(inst *instance.Instance, set *stack.Set, images stack.ImageSet, to string) (*instance.State, []string, error) {
+ cur := inst.State
+ next := *cur
+ next.Release = to
+ next.Upgrade = nil
+ next.Pending = nil
+ next.Settings = maps.Clone(cur.Settings)
+ next.Secrets = maps.Clone(cur.Secrets)
+ next.Rendered = maps.Clone(cur.Rendered)
+ next.Images = instance.Images{Release: to, Refs: maps.Clone(images.Images)}
+ if next.Images.Refs == nil {
+ next.Images.Refs = map[string]string{}
+ }
+ var pins []string
+ if cur.Release == stack.LegacyRelease {
+ pins = slices.Sorted(maps.Keys(instance.ConvertLegacy(&next, set, next.Images.Refs).Pins))
+ }
+ for _, m := range stack.Pending(next.Migrations) {
+ if err := m.Apply(next.Settings); err != nil {
+ return nil, nil, fmt.Errorf("%s: %w", m.ID, err)
+ }
+ }
+ next.Migrations = next.Migrations[:0]
+ for _, m := range stack.Migrations {
+ next.Migrations = append(next.Migrations, m.ID)
+ }
+ if _, err := secrets.Mint(next.Secrets, true); err != nil {
+ return nil, nil, err
+ }
+ return &next, pins, nil
+}
+
+func postgresRef(s *instance.State) string {
+ for _, v := range []string{s.Settings[settings.ImagesPrefix+"postgres"], s.Settings[settings.ImageVars["postgres"]], s.Images.Refs["postgres"]} {
+ if v != "" {
+ return v
+ }
+ }
+ return ""
+}
+
+func refList(refs map[string]string) []string {
+ return slices.Sorted(maps.Values(refs))
+}
+
+func pullRelease(ctx context.Context, app *App, inst *instance.Instance, e *engine.Engine, set *stack.Set, next *instance.State, toShort string) error {
+ scratch := filepath.Join(release.Dir(inst, next.Release), scratchDir)
+ defer os.RemoveAll(scratch)
+ r, err := stack.Render(set, stack.OverlaysFor(next.Settings), next.Images.Refs, inst.Project(), scratch)
+ if err != nil {
+ return ui.Wrap(err, fmt.Sprintf("Couldn't prepare the files for Fluxer %s.", toShort), "Run rexctl upgrade --verbose to see why.")
+ }
+ files := slices.Clone(r.ComposeFiles)
+ for i, f := range files {
+ if !filepath.IsAbs(f) {
+ files[i] = filepath.Join(scratch, f)
+ }
+ }
+ if fileExists(inst.Path(instance.CustomCompose)) {
+ files = append(files, inst.Path(instance.CustomCompose))
+ }
+ var admin map[string]string
+ if next.AdminImport != nil {
+ admin = next.AdminImport.Values
+ }
+ env, err := settings.Effective(set, next.Settings, next.Secrets, admin)
+ if err != nil {
+ return ui.Wrap(err, fmt.Sprintf("Fluxer %s doesn't accept the settings on this machine.", toShort), "Run rexctl upgrade --verbose to see why.")
+ }
+ p, err := e.Load(ctx, engine.Spec{Name: inst.Project(), Dir: inst.Dir, Files: files, Env: env})
+ if err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ return ui.Wrap(engineFailure(err), fmt.Sprintf("Fluxer %s doesn't accept the settings on this machine.", toShort), "Run rexctl upgrade --verbose to see why.")
+ }
+ err = app.UI.Step(ctx, "Downloading Fluxer "+toShort, func(ctx context.Context, s *ui.Step) error {
+ if err := p.Pull(ctx, app.progress(s, "Downloading")); err != nil {
+ return err
+ }
+ s.Done("Downloaded Fluxer %s.", toShort)
+ return nil
+ })
+ if err == nil {
+ return nil
+ }
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ still := "Your instance hasn't changed."
+ if pend := inst.State.Upgrade; pend != nil {
+ fromShort, _ := versionPair(pend.From, pend.To)
+ still = fmt.Sprintf("Your instance is still on %s.", fluxerName(fromShort))
+ }
+ report := engineFailure(err)
+ var ee *ui.ExitError
+ if errors.As(report, &ee) && ee.Err != nil {
+ ee.Msg = fmt.Sprintf("Couldn't download Fluxer %s. %s %s", toShort, ee.Msg, still)
+ return ee
+ }
+ return ui.Wrap(err, fmt.Sprintf("Couldn't download Fluxer %s. %s", toShort, still), "Check this machine's internet connection, then run rexctl upgrade again.")
+}
+
+func upAll(ctx context.Context, p *engine.Project, onEvent func(engine.Event)) error {
+ err := p.Up(context.WithoutCancel(ctx), engine.UpOptions{RemoveOrphans: true, OnEvent: onEvent})
+ if engine.ClassOf(err) != engine.ClassDependencyFailed {
+ return err
+ }
+ return startEach(ctx, p, true)
+}
+
+func startEach(ctx context.Context, p *engine.Project, stopOnCrash bool) error {
+ for _, s := range p.Services() {
+ if stopOnCrash {
+ if err := p.Crashing(ctx); err != nil {
+ return err
+ }
+ }
+ if err := p.Up(context.WithoutCancel(ctx), engine.UpOptions{Services: []string{s}, NoDeps: true}); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func failedStart(ctx context.Context, app *App, inst *instance.Instance, e *engine.Engine, p *engine.Project, waiting []string, cause error) error {
+ pend := inst.State.Upgrade
+ fromShort, toShort := versionPair(pend.From, pend.To)
+ app.UI.Fail(fmt.Sprintf("Fluxer %s didn't start.", toShort), "")
+ app.UI.Detail("%v", cause)
+ logPath := writeFailureLog(ctx, inst, p, "upgrade-"+release.DirName(pend.To)+".log", cause, waiting)
+ if err := goBack(ctx, inst, e, pend.From); err != nil {
+ return err
+ }
+ inst.State.FailedRelease = pend.To
+ _ = inst.Save()
+ app.UI.OK("Went back to %s. Your instance is running.", fluxerName(fromShort))
+ if logPath != "" {
+ fixLine(app, fmt.Sprintf("Read %s to see why.", logPath))
+ } else {
+ fixLine(app, "Run rexctl upgrade --verbose to see why.")
+ }
+ app.SetExit(ui.ExitFailed)
+ return nil
+}
+
+func writeFailureLog(ctx context.Context, inst *instance.Instance, p *engine.Project, name string, cause error, waiting []string) string {
+ ctx = context.WithoutCancel(ctx)
+ if len(waiting) == 0 {
+ waiting = notRunning(ctx, p)
+ }
+ path := inst.Path(instance.StateDirName, name)
+ f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+ if err != nil {
+ return ""
+ }
+ defer f.Close()
+ fmt.Fprintf(f, "rexctl: %v\n", cause)
+ if len(waiting) > 0 {
+ fmt.Fprintf(f, "rexctl: not ready: %s\n", strings.Join(waiting, ", "))
+ }
+ fmt.Fprintln(f)
+ _ = p.Logs(ctx, engine.LogsOptions{Tail: 200, Stdout: f}, waiting...)
+ return path
+}
+
+func writeBackupLog(inst *instance.Instance, name string, cause error) string {
+ text := fmt.Sprintf("rexctl: backup failed: %v\n", cause)
+ var ee *ui.ExitError
+ if errors.As(cause, &ee) && ee.Err != nil && ee.Msg != "" {
+ text += fmt.Sprintf("rexctl: %v\n", ee.Err)
+ }
+ var dump *backup.DumpError
+ if errors.As(cause, &dump) && dump.Output != "" {
+ text += "\n" + dump.Output + "\n"
+ }
+ path := inst.Path(instance.StateDirName, name)
+ if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
+ return ""
+ }
+ return path
+}
+
+func notRunning(ctx context.Context, p *engine.Project) []string {
+ containers, err := p.PS(ctx)
+ if err != nil {
+ return nil
+ }
+ var out []string
+ for _, c := range containers {
+ ok := c.State == "running" && c.Health != "unhealthy" || stack.OneShot[c.Service] && c.State == "exited" && c.ExitCode == 0
+ if !ok && !slices.Contains(out, c.Service) {
+ out = append(out, c.Service)
+ }
+ }
+ return out
+}
+
+func abortUpgrade(ctx context.Context, app *App, inst *instance.Instance, e *engine.Engine, cause, report error) error {
+ pend := inst.State.Upgrade
+ app.UI.Detail("%v", cause)
+ if err := goBack(ctx, inst, e, pend.From); err != nil {
+ var ee *ui.ExitError
+ if errors.As(report, &ee) && ee.Msg != "" {
+ msg, _, _ := strings.Cut(ee.Msg, ". ")
+ app.UI.Fail(strings.TrimSuffix(msg, ".")+".", "")
+ }
+ return err
+ }
+ if ctx.Err() != nil {
+ fromShort, _ := versionPair(pend.From, pend.To)
+ app.UI.Info("Stopped the upgrade. %s is still running.", capitalise(fluxerName(fromShort)))
+ return ctx.Err()
+ }
+ return report
+}
+
+func goBack(ctx context.Context, inst *instance.Instance, e *engine.Engine, version string) error {
+ pend := inst.State.Upgrade
+ cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), recoveryTimeout)
+ defer cancel()
+ if _, err := restoreRelease(cctx, inst, e, version); err != nil {
+ logPath := writeFailureLog(cctx, inst, e.Named(inst.Project(), inst.Dir), "go-back-"+release.DirName(version)+".log", err, nil)
+ fix := "Run rexctl doctor to see what's wrong."
+ if logPath != "" {
+ fix = fmt.Sprintf("Read %s to see why.", logPath)
+ }
+ if pend != nil && pend.Backup != "" {
+ fix = strings.TrimSuffix(fix, ".") + ", or run rexctl rollback --restore-database to restore the backup from before the upgrade."
+ }
+ switch {
+ case pend == nil:
+ case phaseIndex(pend.Phase) < phaseIndex(release.PhaseRecreated):
+ inst.State.Upgrade = nil
+ _ = inst.Save()
+ default:
+ pend.BackFailed = true
+ _ = inst.Save()
+ fix = release.UnfinishedFix(pend)
+ if logPath != "" {
+ fix = fmt.Sprintf("Read %s to see why. %s", logPath, fix)
+ }
+ }
+ return ui.Wrap(err, fmt.Sprintf("Couldn't go back to %s.", fluxerName(version)), fix)
+ }
+ inst.State.Upgrade = nil
+ if err := inst.Save(); err != nil {
+ return ui.Wrap(err, "Couldn't save your instance's state.", "Run rexctl doctor to see what's wrong.")
+ }
+ return nil
+}
+
+func restoreRelease(ctx context.Context, inst *instance.Instance, e *engine.Engine, version string) ([]string, error) {
+ var put []string
+ if version != inst.State.Release {
+ restored, err := release.Restore(inst, version)
+ if err != nil {
+ return nil, err
+ }
+ put = restored.Settings
+ if err := inst.Save(); err != nil {
+ return put, err
+ }
+ }
+ pull := false
+ for _, ref := range inst.State.Images.Refs {
+ if _, found, err := e.LocalImage(ctx, ref); err != nil || !found {
+ pull = true
+ break
+ }
+ }
+ if err := e.Named(inst.Project(), inst.Dir).RemoveReplacements(ctx); err != nil {
+ return put, err
+ }
+ _, err := deploy.Apply(ctx, inst, e, deploy.ApplyOptions{Pull: pull})
+ if err != nil && engine.ClassOf(err) != engine.ClassDependencyFailed {
+ return put, err
+ }
+ p, lerr := deploy.Load(ctx, inst, e)
+ if lerr != nil {
+ return put, lerr
+ }
+ if err != nil {
+ if err := startEach(ctx, p, false); err != nil {
+ return put, err
+ }
+ }
+ started := map[string]bool{}
+ return put, p.WaitReady(ctx, readyTimeout, func(r engine.Readiness) {
+ var stopped []string
+ for _, svc := range r.Failed {
+ if !stack.OneShot[svc] && !started[svc] {
+ started[svc] = true
+ stopped = append(stopped, svc)
+ }
+ }
+ if len(stopped) > 0 {
+ _ = p.Up(ctx, engine.UpOptions{Services: stopped, NoDeps: true})
+ }
+ })
+}
+
+func finishUpgrade(ctx context.Context, app *App, inst *instance.Instance, e *engine.Engine) error {
+ pend := inst.State.Upgrade
+ next, err := release.SnapshotState(inst, pend.To)
+ if err != nil {
+ return ui.Wrap(err, fmt.Sprintf("Couldn't read the settings for Fluxer %s.", release.Short(pend.To)), "Run rexctl doctor to see what's wrong.")
+ }
+ p := e.Named(inst.Project(), inst.Dir)
+ app.sendAdminImport(ctx, inst, p)
+ if done, err := setup.Done(ctx, app.adminClient(inst, p)); err == nil {
+ inst.State.SetupDone = done
+ }
+ inst.State.PreviousRelease = pend.From
+ inst.State.Release = pend.To
+ inst.State.Images = next.Images
+ inst.State.Settings = next.Settings
+ inst.State.Secrets = next.Secrets
+ if ai := inst.State.AdminImport; ai != nil && ai.SentAt != "" {
+ inst.State.AdminImport = nil
+ }
+ inst.State.FailedRelease = ""
+ inst.State.Upgrade = nil
+ if err := inst.Save(); err != nil {
+ return ui.Wrap(err, "Couldn't save your instance's state.", "Run rexctl doctor to see what's wrong.")
+ }
+ if loaded, err := deploy.Load(ctx, inst, e); err == nil {
+ if err := refreshBootUnit(ctx, inst, e, loaded); err != nil {
+ app.UI.Detail("%v", err)
+ }
+ }
+ if err := release.Prune(ctx, inst, e); err != nil {
+ app.UI.Detail("%v", err)
+ }
+ _, toShort := versionPair(pend.From, pend.To)
+ app.UI.OK("Your instance is on Fluxer %s.", toShort)
+ return nil
+}
+
+func recoverUpgrade(ctx context.Context, app *App, inst *instance.Instance, forward bool) error {
+ pend := inst.State.Upgrade
+ if pend == nil {
+ return nil
+ }
+ phase := phaseIndex(pend.Phase)
+ switch {
+ case phase >= phaseIndex(release.PhaseVerified):
+ e, err := app.Engine(ctx, inst)
+ if err != nil {
+ return err
+ }
+ return finishUpgrade(ctx, app, inst, e)
+ case forward && forwardable(inst):
+ e, err := app.Engine(ctx, inst)
+ if err != nil {
+ return err
+ }
+ _, toShort := versionPair(pend.From, pend.To)
+ app.UI.Warn(fmt.Sprintf("The upgrade to Fluxer %s didn't finish.", toShort), "")
+ return retryUpgrade(ctx, app, inst, e)
+ case phase >= phaseIndex(release.PhasePulled) && release.HasSnapshot(inst, pend.From):
+ e, err := app.Engine(ctx, inst)
+ if err != nil {
+ return err
+ }
+ fromShort, toShort := versionPair(pend.From, pend.To)
+ app.UI.Warn(fmt.Sprintf("The upgrade to Fluxer %s didn't finish.", toShort), "")
+ err = app.UI.Step(ctx, "Going back to "+fluxerName(fromShort), func(ctx context.Context, s *ui.Step) error {
+ s.Done("Went back to %s. Your instance is running.", fluxerName(fromShort))
+ return goBack(ctx, inst, e, pend.From)
+ })
+ if err == nil || !forward || !forwardable(inst) || ctx.Err() != nil {
+ return err
+ }
+ var ee *ui.ExitError
+ if errors.As(err, &ee) {
+ app.UI.Fail(ee.Msg, "")
+ }
+ app.UI.Detail("%v", err)
+ return retryUpgrade(ctx, app, inst, e)
+ }
+ clearUpgrade(inst)
+ return nil
+}
+
+func checkEnv(ctx context.Context, inst *instance.Instance, e *engine.Engine, p *engine.Project) *checks.Env {
+ h, _ := host.Detect(ctx, inst.Dir)
+ mode := inst.State.Settings[settings.EdgeMode]
+ return &checks.Env{
+ Dir: inst.Dir,
+ Domain: inst.Domain(),
+ Instance: inst,
+ Engine: e,
+ Project: p,
+ Host: h,
+ Behind: mode == settings.EdgeProxy || mode == settings.EdgeTunnel,
+ }
+}
+
+func resolveRelease(ctx context.Context, to string) (*release.Release, error) {
+ rel, err := release.Resolve(ctx, to)
+ if errors.Is(err, release.ErrUnknownVersion) {
+ return nil, ui.Refused(fmt.Sprintf("There's no Fluxer release called %s.", to), "Run rexctl upgrade --check to see the latest release.")
+ }
+ if err != nil {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ if errors.Is(err, release.ErrBadResponse) {
+ return nil, ui.Wrap(err, "Couldn't check for a new release. Fluxer's release site sent an error.", "Try again later.")
+ }
+ return nil, ui.Wrap(err, "Couldn't check for a new release.", "Check this machine's internet connection, then try again.")
+ }
+ return rel, nil
+}
+
+func tooOldError(err error) error {
+ var e *release.TooOldError
+ if errors.As(err, &e) {
+ return ui.Refused(e.Error(), fmt.Sprintf("Run %s first, then run rexctl upgrade again.", e.Command()))
+ }
+ return err
+}
+
+func selfUpdateError(err error, inst *instance.Instance) error {
+ switch {
+ case errors.Is(err, context.Canceled):
+ return err
+ case errors.Is(err, fs.ErrPermission):
+ fix := fmt.Sprintf("Run sudo rexctl upgrade --dir %s.", inst.Dir)
+ switch {
+ case runtime.GOOS == "windows":
+ fix = "Run rexctl upgrade from a terminal opened as administrator."
+ case inst.State != nil && inst.State.Engine.Kind == string(engine.Podman) && os.Geteuid() != 0:
+ fix = "Move rexctl to a folder your account can write to, like ~/.local/bin, then run rexctl upgrade."
+ }
+ path, _ := release.Executable()
+ return ui.Refused(fmt.Sprintf("rexctl can't replace itself at %s.", path), fix)
+ case errors.Is(err, release.ErrChecksum):
+ return ui.Wrap(err, "The rexctl download didn't match its checksum.", "Run rexctl upgrade again.")
+ case errors.Is(err, release.ErrBadResponse):
+ return ui.Wrap(err, "Couldn't update rexctl. Fluxer's release site sent an error.", "Try rexctl upgrade again later.")
+ }
+ return ui.Wrap(err, "Couldn't update rexctl.", "Check this machine's internet connection, then run rexctl upgrade again.")
+}
+
+func setPhase(inst *instance.Instance, from, to, phase string) error {
+ if inst.State.Upgrade == nil {
+ inst.State.Upgrade = &instance.Upgrade{From: from, To: to, StartedAt: time.Now().UTC()}
+ }
+ inst.State.Upgrade.Phase = phase
+ return inst.Save()
+}
+
+func clearUpgrade(inst *instance.Instance) {
+ inst.State.Upgrade = nil
+ _ = inst.Save()
+}
+
+func phaseIndex(phase string) int {
+ for i, p := range release.Phases {
+ if p == phase {
+ return i
+ }
+ }
+ return -1
+}
+
+func resumeArgs(args []string, phase string) []string {
+ out := make([]string, 0, len(args)+2)
+ flag := "--" + release.ResumeFlag
+ for i := 0; i < len(args); i++ {
+ a := args[i]
+ if a == flag {
+ i++
+ continue
+ }
+ if strings.HasPrefix(a, flag+"=") {
+ continue
+ }
+ out = append(out, a)
+ }
+ return append(out, flag, phase)
+}
+
+func fileExists(path string) bool {
+ _, err := os.Stat(path)
+ return err == nil
+}
+
+func fluxerName(short string) string {
+ if short == "" {
+ return "the Fluxer you ran before rexctl"
+ }
+ return "Fluxer " + short
+}
+
+func versionPair(a, b string) (string, string) {
+ sa, sb := release.Short(a), release.Short(b)
+ if sa == sb {
+ return a, b
+ }
+ return sa, sb
+}
+
+func fixLine(app *App, fix string) {
+ if app.UI.Mode() == ui.ModeJSON {
+ return
+ }
+ fmt.Fprintf(app.UI.Stderr(), " %s\n", fix)
+}