Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

feat(rexctl): import settings from an existing .env

Hampusfcd36d84e974
diff --git a/fluxer_rexctl/internal/instance/envdiff.go b/fluxer_rexctl/internal/instance/envdiff.go
new file mode 100644
index 000000000..0fa802fc5
--- /dev/null
+++ b/fluxer_rexctl/internal/instance/envdiff.go
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package instance
+
+import (
+ "maps"
+ "slices"
+ "strings"
+
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/secrets"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/settings"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack"
+)
+
+type ChangeKind int
+
+const (
+ ChangeSetting ChangeKind = iota
+ ChangeSecret
+ ChangeAdmin
+)
+
+type EnvChange struct {
+ Var string
+ Key string
+ Value string
+ Kind ChangeKind
+ Secret bool
+}
+
+func (c EnvChange) Command() string {
+ if c.Secret {
+ return "rexctl config set " + c.Key + " -"
+ }
+ return "rexctl config set " + c.Key + " " + shellQuote(c.Value)
+}
+
+func shellQuote(v string) string {
+ if v == "" {
+ return "''"
+ }
+ safe := true
+ for _, r := range v {
+ switch {
+ case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
+ case strings.ContainsRune("-_.,:/@+=%", r):
+ default:
+ safe = false
+ }
+ }
+ if safe {
+ return v
+ }
+ return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'"
+}
+
+func compareEnv(set *stack.Set, custom []byte, s *State, env map[string]string) ([]EnvChange, []string, error) {
+ eff, err := effectiveFor(set, s)
+ if err != nil {
+ return nil, nil, err
+ }
+ read, err := settings.Vars(set, custom)
+ if err != nil {
+ return nil, nil, err
+ }
+ defaults := composeDefaults(setFiles(set, custom))
+ named := map[string]settings.Key{}
+ for _, k := range settings.Named() {
+ if len(k.Vars) > 0 {
+ named[k.Vars[0]] = k
+ }
+ }
+ proxies := map[string]string{"HTTP_PROXY": settings.HTTPProxy, "HTTPS_PROXY": settings.HTTPSProxy, "NO_PROXY": settings.NoProxy}
+ admin := twinsToAdmin(maps.Clone(env), true)
+ var out []EnvChange
+ var unused []string
+ seen := map[string]bool{}
+ for _, k := range slices.Sorted(maps.Keys(env)) {
+ v := env[k]
+ if strings.HasPrefix(k, "COMPOSE_") {
+ continue
+ }
+ cur, ok := eff[k]
+ if v == cur || (!ok && (v == "" || equalsDefault(defaults[k], v, eff))) {
+ continue
+ }
+ c := EnvChange{Var: k, Key: k, Value: v}
+ switch {
+ case isGeneratedSecret(k):
+ c.Secret = k != secrets.VAPIDPublicKey
+ if _, chosen := s.Settings[k]; !chosen {
+ c.Kind = ChangeSecret
+ }
+ case settings.Twins[k] != "":
+ c.Kind, c.Key = ChangeAdmin, settings.Twins[k]
+ if translated, ok := admin[c.Key]; ok {
+ c.Value = translated
+ }
+ c.Secret = settings.SecretName(k)
+ case proxies[strings.ToUpper(k)] != "":
+ c.Key = proxies[strings.ToUpper(k)]
+ c.Secret = hasUserinfo(v)
+ case named[k].Name != "":
+ c.Key, c.Secret = named[k].Name, named[k].Secret
+ case slices.Contains(read, k):
+ c.Secret = settings.SecretName(k) || hasUserinfo(v)
+ default:
+ unused = append(unused, k)
+ continue
+ }
+ if seen[c.Key] {
+ continue
+ }
+ seen[c.Key] = true
+ out = append(out, c)
+ }
+ return out, unused, nil
+}
+
+func hasUserinfo(v string) bool {
+ scheme, rest, ok := strings.Cut(v, "://")
+ if !ok || scheme == "" {
+ return false
+ }
+ host, _, _ := strings.Cut(rest, "/")
+ return strings.Contains(host, "@")
+}
+
+func applyChanges(s *State, changes []EnvChange) {
+ for _, c := range changes {
+ switch c.Kind {
+ case ChangeSecret:
+ s.Secrets[c.Key] = c.Value
+ case ChangeAdmin:
+ if s.AdminImport == nil {
+ s.AdminImport = &AdminImport{Values: map[string]string{}}
+ }
+ if s.AdminImport.Values == nil {
+ s.AdminImport.Values = map[string]string{}
+ }
+ s.AdminImport.Values[c.Key] = c.Value
+ s.AdminImport.SentAt = ""
+ default:
+ s.Settings[c.Key] = c.Value
+ }
+ }
+}
diff --git a/fluxer_rexctl/internal/instance/import.go b/fluxer_rexctl/internal/instance/import.go
new file mode 100644
index 000000000..531b429d7
--- /dev/null
+++ b/fluxer_rexctl/internal/instance/import.go
@@ -0,0 +1,509 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package instance
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "maps"
+ "os"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "testing/fstest"
+ "time"
+
+ "go.yaml.in/yaml/v4"
+
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/engine"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/secrets"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack"
+)
+
+type Source string
+
+const (
+ SourceInstaller Source = "installer"
+ SourceEjected Source = "ejected"
+ SourceManaged Source = "managed"
+)
+
+const EjectedEnvFile = "ejected.env"
+
+type ImportReport struct {
+ Source Source
+ Dir string
+ Project string
+ PodmanCompose bool
+ CustomFiles []string
+ KeepAsMine []string
+ Mine map[string]string
+ DeleteFiles []string
+ Minted []string
+ Placeholders []string
+ Dropped []string
+ Defaults []string
+ AdminKeys []string
+ Changed []string
+ DedupedEnv bool
+ EngineProxies bool
+ RemovedEnv bool
+}
+
+func (r *ImportReport) DroppedLine() string {
+ if len(r.Dropped) == 0 {
+ return ""
+ }
+ saved := "They are"
+ if len(r.Dropped) == 1 {
+ saved = "It's"
+ }
+ return fmt.Sprintf("rexctl left out %s Fluxer doesn't read: %s. %s saved in %s.",
+ count(len(r.Dropped), "setting", "settings"), strings.Join(r.Dropped, ", "), saved,
+ filepath.Join(StateDirName, ReleasesDir, LegacyRelease, EnvFileName))
+}
+
+func count(n int, one, many string) string {
+ if n == 1 {
+ return "1 " + one
+ }
+ return strconv.Itoa(n) + " " + many
+}
+
+func Import(dir string, e *engine.Engine) (*State, *ImportReport, error) {
+ return runImport(context.Background(), dir, e, false)
+}
+
+func PreviewImport(dir string, e *engine.Engine) (*ImportReport, error) {
+ _, r, err := runImport(context.Background(), dir, e, true)
+ return r, err
+}
+
+func runImport(ctx context.Context, dir string, e *engine.Engine, dry bool) (*State, *ImportReport, error) {
+ abs, err := filepath.Abs(dir)
+ if err != nil {
+ return nil, nil, err
+ }
+ statePath := filepath.Join(abs, StateDirName, StateFileName)
+ data, err := os.ReadFile(statePath)
+ if errors.Is(err, fs.ErrNotExist) {
+ return importInstaller(ctx, abs, e, dry)
+ }
+ if err != nil {
+ return nil, nil, err
+ }
+ s, err := ParseState(data, abs, filepath.Join(abs, EnvFileName))
+ if err != nil {
+ return nil, nil, err
+ }
+ switch {
+ case s.MigratedFromV1():
+ return importRexctl(ctx, abs, e, s, dry)
+ case s.Ejected != nil:
+ return importEjected(ctx, abs, e, s, dry)
+ }
+ r := &ImportReport{Source: SourceManaged, Dir: abs, Project: s.Project}
+ inst := &Instance{Dir: abs, State: s}
+ if d, err := envDrift(inst); err == nil && d != nil && d.Same && !dry {
+ if err := removeIfExists(inst.Path(EnvFileName)); err != nil {
+ return nil, nil, err
+ }
+ r.RemovedEnv = true
+ }
+ return s, r, nil
+}
+
+func refusal(what string) *AdoptError {
+ return &AdoptError{
+ Msg: "rexctl can't manage this setup.",
+ Fix: "Keep running it with docker compose, or " + what + " and run rexctl takeover again.",
+ }
+}
+
+func importInstaller(ctx context.Context, dir string, e *engine.Engine, dry bool) (*State, *ImportReport, error) {
+ l, err := DetectLegacy(dir)
+ if err != nil {
+ return nil, nil, err
+ }
+ p, err := PlanAdopt(l)
+ if err != nil {
+ return nil, nil, err
+ }
+ r := &ImportReport{
+ Source: SourceInstaller,
+ Dir: dir,
+ Project: p.Project,
+ CustomFiles: p.CustomFiles,
+ KeepAsMine: p.KeepAsMine,
+ Mine: p.Mine,
+ DeleteFiles: p.DeleteFiles,
+ DedupedEnv: p.DedupedEnv,
+ }
+ var custom []byte
+ if p.customFrom != "" {
+ if custom, err = l.read(p.customFrom); err != nil {
+ return nil, nil, err
+ }
+ if what := customProblem(custom); what != "" {
+ return nil, nil, refusal(fmt.Sprintf("remove %s from %s", what, displayPath(dir, p.customFrom)))
+ }
+ }
+ live, err := inspect(ctx, e, p.Project, dir)
+ if err != nil {
+ return nil, nil, err
+ }
+ r.PodmanCompose = live.podmanCompose
+ if live.podmanCompose && e.Kind == engine.Podman && VersionBelow(e.Version, engine.MinPodman) {
+ return nil, nil, &AdoptError{
+ Msg: "rexctl can't manage this setup.",
+ Fix: fmt.Sprintf("Keep running it with podman-compose, or upgrade Podman to %s or newer and run rexctl takeover again.", engine.MinPodman),
+ }
+ }
+ set, err := legacySet(l)
+ if err != nil {
+ return nil, nil, err
+ }
+ examples := map[string]string{}
+ if data, err := l.read(filepath.Join(dir, ".env.example")); err == nil {
+ if m, _, err := parseEnv(data); err == nil {
+ examples = m
+ }
+ }
+ envData, err := l.read(filepath.Join(dir, EnvFileName))
+ if err != nil {
+ return nil, nil, err
+ }
+ env, _, err := parseEnv(envData)
+ if err != nil {
+ return nil, nil, err
+ }
+ overlays := make([]stack.Overlay, 0, len(p.Overlays))
+ for _, o := range p.Overlays {
+ overlays = append(overlays, stack.Overlay(o))
+ }
+ vals, err := importValues(valueInput{
+ env: env, set: set, custom: custom, overlays: overlays, legacy: true,
+ examples: examples, proxies: live.proxies,
+ })
+ if err != nil {
+ return nil, nil, err
+ }
+ minted, err := mint(vals.secrets, live.haveVolumes)
+ if err != nil {
+ return nil, nil, err
+ }
+ fill(r, vals, minted)
+ if dry {
+ return nil, r, nil
+ }
+ if err := adoptFiles(ctx, p); err != nil {
+ return nil, nil, err
+ }
+ installed := time.Now().UTC().Truncate(time.Second)
+ if fi, err := os.Stat(filepath.Join(l.snapshotDir(), EnvFileName)); err == nil {
+ installed = fi.ModTime().UTC().Truncate(time.Second)
+ }
+ s := &State{
+ Schema: StateSchema,
+ Project: p.Project,
+ Release: LegacyRelease,
+ Images: Images{Release: LegacyRelease, Refs: live.images},
+ Settings: vals.settings,
+ Secrets: vals.secrets,
+ SetupDone: true,
+ InstalledAt: installed,
+ UpdatedAt: time.Now().UTC().Truncate(time.Second),
+ }
+ if len(vals.admin) > 0 {
+ s.AdminImport = &AdminImport{Values: vals.admin}
+ }
+ pin(s, e)
+ s.normalize()
+ if err := finishImport(dir, s); err != nil {
+ return nil, nil, err
+ }
+ return s, r, nil
+}
+
+func finishImport(dir string, s *State) error {
+ if err := writeReleaseSet(dir, s.Release); err != nil {
+ return err
+ }
+ if err := recordRendered(dir, s); err != nil {
+ return err
+ }
+ s.UpdatedAt = time.Now().UTC().Truncate(time.Second)
+ if err := s.Save(filepath.Join(dir, StateDirName, StateFileName)); err != nil {
+ return err
+ }
+ return removeIfExists(filepath.Join(dir, EnvFileName))
+}
+
+func fill(r *ImportReport, vals *valueResult, minted []string) {
+ r.Minted = minted
+ r.Placeholders = vals.placeholders
+ r.Dropped = vals.dropped
+ r.Defaults = vals.defaults
+ r.AdminKeys = slices.Sorted(maps.Keys(vals.admin))
+ r.EngineProxies = vals.engineProxies
+}
+
+func pin(s *State, e *engine.Engine) {
+ if e == nil {
+ return
+ }
+ s.Engine = EngineRef{Kind: string(e.Kind), Host: e.Host, ID: e.ID}
+}
+
+func mint(values map[string]string, haveVolumes bool) ([]string, error) {
+ minted, err := secrets.Mint(values, haveVolumes)
+ var missing *secrets.StatefulMissingError
+ if errors.As(err, &missing) {
+ return nil, &AdoptError{
+ Msg: fmt.Sprintf("Your instance's data needs %s, and rexctl can't find %s.", strings.Join(missing.Keys, ", "), pick(len(missing.Keys), "its value", "their values")),
+ Fix: "Restore the instance from a backup with rexctl restore.",
+ }
+ }
+ return minted, err
+}
+
+func pick(n int, one, many string) string {
+ if n == 1 {
+ return one
+ }
+ return many
+}
+
+type liveState struct {
+ haveVolumes bool
+ podmanCompose bool
+ images map[string]string
+ proxies engine.Proxies
+}
+
+func inspect(ctx context.Context, e *engine.Engine, project, dir string) (*liveState, error) {
+ live := &liveState{haveVolumes: true, images: map[string]string{}}
+ if e == nil {
+ return live, nil
+ }
+ live.proxies = e.Proxies()
+ vols, err := e.Volumes(ctx, project)
+ if err != nil {
+ return nil, err
+ }
+ live.haveVolumes = len(vols) > 0
+ containers, err := e.Named(project, dir).PS(ctx)
+ if err != nil {
+ return nil, err
+ }
+ for _, c := range containers {
+ if c.Labels[engine.PodmanComposeLabel] != "" {
+ live.podmanCompose = true
+ }
+ if c.Service == "" || c.Image == "" {
+ continue
+ }
+ live.images[c.Service] = c.Image
+ if strings.Contains(c.Image, "@") || c.ImageID == "" {
+ continue
+ }
+ if d, err := e.RepoDigest(ctx, c.ImageID, c.Image); err == nil && d != "" {
+ live.images[c.Service] = d
+ }
+ }
+ return live, nil
+}
+
+func VersionBelow(version, floor string) bool {
+ parse := func(v string) []int {
+ v = strings.TrimPrefix(strings.TrimSpace(v), "v")
+ v, _, _ = strings.Cut(v, "-")
+ var out []int
+ for _, part := range strings.SplitN(v, ".", 3) {
+ n, err := strconv.Atoi(part)
+ if err != nil {
+ return nil
+ }
+ out = append(out, n)
+ }
+ return out
+ }
+ a, b := parse(version), parse(floor)
+ if a == nil || b == nil {
+ return false
+ }
+ for i := range b {
+ x := 0
+ if i < len(a) {
+ x = a[i]
+ }
+ if x != b[i] {
+ return x < b[i]
+ }
+ }
+ return false
+}
+
+func customProblem(data []byte) string {
+ var doc map[string]any
+ if err := yaml.Unmarshal(data, &doc); err != nil {
+ return ""
+ }
+ if _, ok := doc["include"]; ok {
+ return "include"
+ }
+ services, _ := doc["services"].(map[string]any)
+ for _, name := range slices.Sorted(maps.Keys(services)) {
+ svc, _ := services[name].(map[string]any)
+ if _, ok := svc["build"]; ok {
+ return "build"
+ }
+ if _, ok := svc["env_file"]; ok {
+ return "env_file"
+ }
+ switch ext := svc["extends"].(type) {
+ case map[string]any:
+ if _, ok := ext["file"]; ok {
+ return "extends"
+ }
+ }
+ }
+ return ""
+}
+
+func legacySet(l *Legacy) (*stack.Set, error) {
+ files := fstest.MapFS{}
+ for i, f := range l.ComposeFiles {
+ data, err := l.read(f)
+ if err != nil {
+ continue
+ }
+ name := filepath.Base(f)
+ if i == 0 {
+ name = stack.ComposeFile
+ } else if overlayFor(f, l.Dir) == "" {
+ continue
+ }
+ files[name] = &fstest.MapFile{Data: normalizeLF(data), Mode: 0o444}
+ }
+ if data, err := l.read(filepath.Join(l.Dir, stack.Caddyfile)); err == nil {
+ files[stack.Caddyfile] = &fstest.MapFile{Data: normalizeLF(data), Mode: 0o444}
+ }
+ return newSet(LegacyRelease, files, nil)
+}
+
+func newSet(release string, files fs.FS, profiles map[string]map[string]string) (*stack.Set, error) {
+ vars, err := stack.FilesVars(files)
+ if err != nil {
+ return nil, err
+ }
+ if profiles == nil {
+ profiles = map[string]map[string]string{}
+ }
+ return &stack.Set{Release: release, Files: files, Profiles: profiles, Vars: vars}, nil
+}
+
+func renderedNames(dir string) []string {
+ names := []string{stack.ComposeFile}
+ for _, o := range slices.Sorted(maps.Keys(stack.OverlayFiles)) {
+ names = append(names, stack.OverlayFiles[o])
+ }
+ names = append(names, stack.Caddyfile)
+ var out []string
+ for _, n := range names {
+ if isFile(filepath.Join(dir, n)) {
+ out = append(out, n)
+ }
+ }
+ return out
+}
+
+func diskSet(dir, release string, profiles map[string]map[string]string) (*stack.Set, error) {
+ files := fstest.MapFS{}
+ for _, name := range renderedNames(dir) {
+ data, err := os.ReadFile(filepath.Join(dir, name))
+ if err != nil {
+ return nil, err
+ }
+ files[name] = &fstest.MapFile{Data: data, Mode: 0o444}
+ }
+ if _, ok := files[stack.ComposeFile]; !ok {
+ return nil, fmt.Errorf("%w: %s", ErrNoReleaseSet, release)
+ }
+ return newSet(release, files, profiles)
+}
+
+func writeReleaseSet(dir, release string) error {
+ target := filepath.Join(dir, StateDirName, ReleasesDir, release, StackDir)
+ if isDir(target) {
+ return nil
+ }
+ var set *stack.Set
+ if embedded := stack.Embedded(); release == embedded.Release {
+ set = embedded
+ } else {
+ var profiles map[string]map[string]string
+ if release != LegacyRelease {
+ profiles = stack.Embedded().Profiles
+ }
+ var err error
+ if set, err = diskSet(dir, release, profiles); err != nil {
+ return err
+ }
+ }
+ return stack.WriteSet(target, set)
+}
+
+func recordRendered(dir string, s *State) error {
+ if len(s.Rendered) > 0 {
+ return nil
+ }
+ if s.Rendered == nil {
+ s.Rendered = map[string]string{}
+ }
+ for _, name := range renderedNames(dir) {
+ data, err := os.ReadFile(filepath.Join(dir, name))
+ if err != nil {
+ return err
+ }
+ s.Rendered[name] = stack.Hash(data)
+ }
+ return nil
+}
+
+func releaseSet(dir, release string) (*stack.Set, error) {
+ set, err := stack.LoadSet(filepath.Join(dir, StateDirName, ReleasesDir, release, StackDir))
+ if err == nil {
+ return set, nil
+ }
+ if embedded := stack.Embedded(); release == embedded.Release {
+ return embedded, nil
+ }
+ var profiles map[string]map[string]string
+ if release != LegacyRelease {
+ profiles = stack.Embedded().Profiles
+ }
+ return diskSet(filesDir, release, profiles)
+}
+
+func readCustom(dir string) []byte {
+ data, _ := os.ReadFile(filepath.Join(dir, CustomCompose))
+ return data
+}
+
+func otherEngine(pinned, connected string) *AdoptError {
+ return refusal(fmt.Sprintf("connect rexctl to %s instead of %s", engineName(pinned), engineName(connected)))
+}
+
+func engineName(kind string) string {
+ switch kind {
+ case string(engine.Podman):
+ return "Podman"
+ case string(engine.Docker):
+ return "Docker"
+ }
+ return "the engine that runs it"
+}