Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

feat(rexctl): restore instance backups

Hampusac1157ed1a1b
diff --git a/fluxer_rexctl/internal/backup/restore.go b/fluxer_rexctl/internal/backup/restore.go
new file mode 100644
index 000000000..c1d267670
--- /dev/null
+++ b/fluxer_rexctl/internal/backup/restore.go
@@ -0,0 +1,511 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package backup
+
+import (
+ "archive/tar"
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "maps"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/deploy"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/engine"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/envfile"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/instance"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/release"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack"
+)
+
+const postgresReadyTimeout = 3 * time.Minute
+
+var errStop = errors.New("backup: stop")
+
+type RestoreOptions struct {
+ Progress func(stage string)
+ OnEvent func(engine.Event)
+}
+
+type ProjectInUseError struct {
+ Project string
+ Dir string
+}
+
+func (e *ProjectInUseError) Error() string {
+ return fmt.Sprintf("backup: project %s is in use by %s", e.Project, e.Dir)
+}
+
+type RestoreError struct {
+ Part string
+ Output string
+ Err error
+}
+
+func (e *RestoreError) Error() string {
+ return fmt.Sprintf("backup: restoring %s: %v", e.Part, e.Err)
+}
+
+func (e *RestoreError) Unwrap() error { return e.Err }
+
+type PullError struct {
+ Err error
+}
+
+func (e *PullError) Error() string {
+ return fmt.Sprintf("backup: downloading images: %v", e.Err)
+}
+
+func (e *PullError) Unwrap() error { return e.Err }
+
+func Restore(ctx context.Context, path, dir string, e *engine.Engine, opts RestoreOptions) (*instance.Instance, error) {
+ progress := opts.Progress
+ if progress == nil {
+ progress = func(string) {}
+ }
+ progress(StageVerifying)
+ m, err := Verify(path)
+ if err != nil {
+ return nil, err
+ }
+ dir, err = filepath.Abs(dir)
+ if err != nil {
+ return nil, err
+ }
+ existing, err := instance.Find(dir)
+ if err != nil && !errors.Is(err, instance.ErrNotFound) {
+ return nil, err
+ }
+ if err := checkProjectFree(ctx, e, m.Project, dir); err != nil {
+ return nil, err
+ }
+
+ progress(StagePulling)
+ if err := pullFromBackup(ctx, path, e, opts.OnEvent); err != nil {
+ return nil, err
+ }
+
+ progress(StageClearing)
+ var stale []string
+ if existing != nil {
+ if existing.State != nil {
+ stale = slices.Collect(maps.Values(existing.State.Images.Refs))
+ }
+ stale = append(stale, release.KeptImages(existing)...)
+ if err := e.Named(existing.Project(), existing.Dir).Down(ctx, true); err != nil {
+ return nil, err
+ }
+ if err := clearInstanceFiles(dir); err != nil {
+ return nil, err
+ }
+ }
+
+ progress(StageFiles)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return nil, err
+ }
+ if err := extractFiles(path, dir); err != nil {
+ return nil, err
+ }
+ inst, err := instance.Find(dir)
+ if err != nil {
+ return nil, err
+ }
+ if err := adoptRestored(inst, e); err != nil {
+ return nil, err
+ }
+ for _, d := range []string{inst.StateDir(), inst.BackupDir()} {
+ if err := os.MkdirAll(d, 0o700); err != nil {
+ return nil, err
+ }
+ if err := envfile.RestrictToOwner(d); err != nil {
+ return nil, err
+ }
+ }
+
+ p, err := deploy.Load(ctx, inst, e)
+ if err != nil {
+ return nil, &RestoreError{Part: "containers", Err: err}
+ }
+ if err := p.Down(ctx, true); err != nil {
+ return nil, err
+ }
+ if err := p.Create(ctx); err != nil {
+ return nil, &RestoreError{Part: "containers", Err: err}
+ }
+
+ if m.Database != "" {
+ progress(StageDatabase)
+ if err := startPostgres(ctx, p); err != nil {
+ return nil, err
+ }
+ }
+ vols, err := e.Volumes(ctx, p.Name)
+ if err != nil {
+ return nil, err
+ }
+ image, err := helperImage(ctx, inst, p)
+ if err != nil {
+ return nil, err
+ }
+ if err := restoreData(ctx, path, p, image, m, vols, progress); err != nil {
+ return nil, err
+ }
+
+ progress(StageStarting)
+ if _, err := deploy.Apply(ctx, inst, e, deploy.ApplyOptions{}); err != nil {
+ return nil, err
+ }
+ removeStaleImages(ctx, p, stale)
+ return inst, nil
+}
+
+func adoptRestored(inst *instance.Instance, e *engine.Engine) error {
+ st := inst.State
+ if st == nil {
+ return fmt.Errorf("%w: no state in the backup", ErrDamaged)
+ }
+ st.Engine = instance.EngineRef{Kind: string(e.Kind), Host: e.Host, ID: e.ID}
+ st.Stopped = false
+ st.Installing = false
+ st.Ejected = nil
+ st.Upgrade = nil
+ st.Pending = nil
+ if prev := st.PreviousRelease; prev != "" {
+ if _, err := os.Stat(inst.ReleaseDir(prev)); err != nil {
+ st.PreviousRelease = ""
+ }
+ }
+ rendered, err := renderRestored(inst)
+ if err != nil {
+ return err
+ }
+ st.Rendered = rendered
+ return inst.Save()
+}
+
+func renderRestored(inst *instance.Instance) (map[string]string, error) {
+ st := inst.State
+ if st.Release != stack.LegacyRelease {
+ if set, err := release.LoadSet(inst, st.Release); err == nil {
+ r, err := stack.Render(set, stack.OverlaysFor(st.Settings), st.Images.Refs, inst.Project(), inst.Dir)
+ if err != nil {
+ return nil, err
+ }
+ return r.Hashes, nil
+ }
+ }
+ out := map[string]string{}
+ for name := range st.Rendered {
+ data, err := os.ReadFile(filepath.Join(inst.Dir, filepath.FromSlash(name)))
+ if err != nil {
+ continue
+ }
+ out[name] = stack.Hash(data)
+ }
+ return out, nil
+}
+
+func removeStaleImages(ctx context.Context, p *engine.Project, stale []string) {
+ current := p.Images()
+ var drop []string
+ for _, r := range stale {
+ if !slices.Contains(current, r) && !slices.Contains(drop, r) {
+ drop = append(drop, r)
+ }
+ }
+ if len(drop) > 0 {
+ _ = p.Engine.RemoveImages(ctx, drop)
+ }
+}
+
+func pullFromBackup(ctx context.Context, path string, e *engine.Engine, onEvent func(engine.Event)) error {
+ staging, err := os.MkdirTemp("", "rexctl-restore-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(staging)
+ if err := extractFiles(path, staging); err != nil {
+ return err
+ }
+ inst, err := instance.Find(staging)
+ if err != nil {
+ return err
+ }
+ p, err := deploy.Load(ctx, inst, e)
+ if err != nil {
+ return &PullError{Err: err}
+ }
+ if err := p.Pull(ctx, onEvent); err != nil {
+ return &PullError{Err: err}
+ }
+ return nil
+}
+
+func checkProjectFree(ctx context.Context, e *engine.Engine, project, dir string) error {
+ projects, err := e.Projects(ctx)
+ if err != nil {
+ return err
+ }
+ for _, pr := range projects {
+ if pr.Name != project || pr.ConfigFiles == "" {
+ continue
+ }
+ first, _, _ := strings.Cut(pr.ConfigFiles, ",")
+ other := filepath.Dir(strings.TrimSpace(first))
+ if !samePath(other, dir) {
+ return &ProjectInUseError{Project: project, Dir: other}
+ }
+ }
+ return nil
+}
+
+func samePath(a, b string) bool {
+ ra, err1 := filepath.EvalSymlinks(a)
+ rb, err2 := filepath.EvalSymlinks(b)
+ if err1 == nil && err2 == nil {
+ return ra == rb
+ }
+ return filepath.Clean(a) == filepath.Clean(b)
+}
+
+func clearInstanceFiles(dir string) error {
+ names, err := instanceFiles(dir)
+ if err != nil {
+ return err
+ }
+ for _, name := range names {
+ if err := os.Remove(filepath.Join(dir, name)); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ }
+ for _, p := range []string{
+ filepath.Join(dir, instance.CertsDir),
+ filepath.Join(dir, instance.CaddyDropInDir),
+ filepath.Join(dir, instance.StateDirName, instance.ReleasesDir),
+ filepath.Join(dir, instance.StateDirName, instance.StateFileName),
+ } {
+ if err := os.RemoveAll(p); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func extractFiles(path, dir string) error {
+ err := walkArchive(path, func(hdr *tar.Header, r io.Reader) error {
+ if !strings.HasPrefix(hdr.Name, FilesPrefix) {
+ return errStop
+ }
+ rel := strings.TrimSuffix(strings.TrimPrefix(hdr.Name, FilesPrefix), "/")
+ if rel == "" {
+ return nil
+ }
+ if !filepath.IsLocal(filepath.FromSlash(rel)) {
+ return fmt.Errorf("%w: unsafe path %q", ErrDamaged, hdr.Name)
+ }
+ target := filepath.Join(dir, filepath.FromSlash(rel))
+ switch hdr.Typeflag {
+ case tar.TypeDir:
+ return os.MkdirAll(target, dirMode(hdr.Mode))
+ case tar.TypeReg:
+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+ return err
+ }
+ return writeFile(target, r, os.FileMode(hdr.Mode).Perm(), rel == instance.EnvFileName)
+ }
+ return nil
+ })
+ if errors.Is(err, errStop) {
+ return nil
+ }
+ return err
+}
+
+func dirMode(mode int64) os.FileMode {
+ m := os.FileMode(mode).Perm()
+ if m == 0 {
+ return 0o755
+ }
+ return m | 0o700
+}
+
+func writeFile(target string, r io.Reader, mode os.FileMode, private bool) error {
+ if private {
+ data, err := io.ReadAll(r)
+ if err != nil {
+ return err
+ }
+ return envfile.WriteFileAtomic(target, data, true)
+ }
+ if mode == 0 {
+ mode = 0o644
+ }
+ tmp, err := os.CreateTemp(filepath.Dir(target), "."+filepath.Base(target)+".*")
+ if err != nil {
+ return err
+ }
+ name := tmp.Name()
+ defer os.Remove(name)
+ if _, err := io.Copy(tmp, r); err != nil {
+ tmp.Close()
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ return err
+ }
+ if err := os.Chmod(name, mode); err != nil {
+ return err
+ }
+ return os.Rename(name, target)
+}
+
+func startPostgres(ctx context.Context, p *engine.Project) error {
+ if err := p.Start(ctx, postgresService); err != nil {
+ return err
+ }
+ ctx, cancel := context.WithTimeout(ctx, postgresReadyTimeout)
+ defer cancel()
+ var stderr bytes.Buffer
+ err := p.Exec(ctx, postgresService, engine.ExecOptions{Stderr: &stderr, KillOnCancel: true}, "sh", "-c",
+ `until pg_isready -q -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"; do sleep 1; done`)
+ if err != nil {
+ return &RestoreError{Part: "database", Output: strings.TrimSpace(stderr.String()), Err: err}
+ }
+ return nil
+}
+
+type volumeSink struct {
+ name string
+ pw *io.PipeWriter
+ tw *tar.Writer
+ done chan error
+ out *bytes.Buffer
+}
+
+func restoreData(ctx context.Context, path string, p *engine.Project, image string, m *Manifest, vols []engine.Volume, progress func(string)) error {
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ var sink *volumeSink
+ closeSink := func(failed error) error {
+ if sink == nil {
+ return nil
+ }
+ s := sink
+ sink = nil
+ if failed != nil {
+ s.pw.CloseWithError(failed)
+ <-s.done
+ return failed
+ }
+ werr := s.tw.Close()
+ s.pw.CloseWithError(werr)
+ rerr := <-s.done
+ if werr != nil {
+ return werr
+ }
+ if rerr != nil {
+ return &RestoreError{Part: s.name, Output: strings.TrimSpace(s.out.String()), Err: rerr}
+ }
+ return nil
+ }
+ openSink := func(short string) error {
+ full, ok := findVolume(vols, p.Name, short)
+ if !ok {
+ return &RestoreError{Part: short, Err: errors.New("the volume wasn't created")}
+ }
+ pr, pw := io.Pipe()
+ s := &volumeSink{name: short, pw: pw, tw: tar.NewWriter(pw), done: make(chan error, 1), out: &bytes.Buffer{}}
+ go func() {
+ err := p.Helper(ctx, engine.HelperOptions{
+ Image: image,
+ Entrypoint: []string{"sh", "-c"},
+ Command: []string{"find /data -mindepth 1 -delete && exec tar -xf - -C /data"},
+ Mounts: []engine.Mount{{Volume: full, Target: "/data"}},
+ Stdin: pr,
+ Stderr: s.out,
+ })
+ if err == nil {
+ io.Copy(io.Discard, pr)
+ }
+ pr.CloseWithError(err)
+ s.done <- err
+ }()
+ sink = s
+ return nil
+ }
+ volumesStarted := false
+ err := walkArchive(path, func(hdr *tar.Header, r io.Reader) error {
+ if hdr.Name == DumpName {
+ if err := closeSink(nil); err != nil {
+ return err
+ }
+ return restoreDump(ctx, p, r)
+ }
+ short, inner, ok := volumeOf(hdr.Name)
+ if !ok {
+ return closeSink(nil)
+ }
+ if !contains(m.Volumes, short) {
+ return fmt.Errorf("%w: %s isn't in the manifest", ErrDamaged, short)
+ }
+ if !volumesStarted {
+ volumesStarted = true
+ progress(StageVolumes)
+ }
+ if sink == nil || sink.name != short {
+ if err := closeSink(nil); err != nil {
+ return err
+ }
+ if err := openSink(short); err != nil {
+ return err
+ }
+ }
+ out := *hdr
+ out.Name = "./" + inner
+ if inner == "." {
+ out.Name = "./"
+ }
+ if hdr.Typeflag == tar.TypeDir && !strings.HasSuffix(out.Name, "/") {
+ out.Name += "/"
+ }
+ if hdr.Typeflag == tar.TypeLink {
+ _, target, ok := volumeOf(hdr.Linkname)
+ if !ok {
+ return fmt.Errorf("%w: unsafe link %q", ErrDamaged, hdr.Linkname)
+ }
+ out.Linkname = "./" + target
+ }
+ out.PAXRecords = nil
+ out.Format = tar.FormatPAX
+ if err := sink.tw.WriteHeader(&out); err != nil {
+ return err
+ }
+ if hdr.Typeflag == tar.TypeReg {
+ if _, err := io.Copy(sink.tw, r); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return closeSink(err)
+ }
+ return closeSink(nil)
+}
+
+func restoreDump(ctx context.Context, p *engine.Project, r io.Reader) error {
+ var stderr bytes.Buffer
+ err := p.Exec(ctx, postgresService, engine.ExecOptions{Stdin: r, Stderr: &stderr, KillOnCancel: true}, "sh", "-c",
+ `exec pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --exit-on-error --single-transaction`)
+ if err != nil {
+ return &RestoreError{Part: "database", Output: strings.TrimSpace(stderr.String()), Err: err}
+ }
+ return nil
+}