A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(rexctl): import settings from an existing .env
Hampusfcd36d84e974
diff --git a/fluxer_rexctl/internal/instance/envdiff.go b/fluxer_rexctl/internal/instance/envdiff.gonew file mode 100644index 000000000..0fa802fc5--- /dev/null+++ b/fluxer_rexctl/internal/instance/envdiff.go@@ -0,0 +1,147 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++package instance++import (+ "maps"+ "slices"+ "strings"++ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/secrets"+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/settings"+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack"+)++type ChangeKind int++const (+ ChangeSetting ChangeKind = iota+ ChangeSecret+ ChangeAdmin+)++type EnvChange struct {+ Var string+ Key string+ Value string+ Kind ChangeKind+ Secret bool+}++func (c EnvChange) Command() string {+ if c.Secret {+ return "rexctl config set " + c.Key + " -"+ }+ return "rexctl config set " + c.Key + " " + shellQuote(c.Value)+}++func shellQuote(v string) string {+ if v == "" {+ return "''"+ }+ safe := true+ for _, r := range v {+ switch {+ case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':+ case strings.ContainsRune("-_.,:/@+=%", r):+ default:+ safe = false+ }+ }+ if safe {+ return v+ }+ return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'"+}++func compareEnv(set *stack.Set, custom []byte, s *State, env map[string]string) ([]EnvChange, []string, error) {+ eff, err := effectiveFor(set, s)+ if err != nil {+ return nil, nil, err+ }+ read, err := settings.Vars(set, custom)+ if err != nil {+ return nil, nil, err+ }+ defaults := composeDefaults(setFiles(set, custom))+ named := map[string]settings.Key{}+ for _, k := range settings.Named() {+ if len(k.Vars) > 0 {+ named[k.Vars[0]] = k+ }+ }+ proxies := map[string]string{"HTTP_PROXY": settings.HTTPProxy, "HTTPS_PROXY": settings.HTTPSProxy, "NO_PROXY": settings.NoProxy}+ admin := twinsToAdmin(maps.Clone(env), true)+ var out []EnvChange+ var unused []string+ seen := map[string]bool{}+ for _, k := range slices.Sorted(maps.Keys(env)) {+ v := env[k]+ if strings.HasPrefix(k, "COMPOSE_") {+ continue+ }+ cur, ok := eff[k]+ if v == cur || (!ok && (v == "" || equalsDefault(defaults[k], v, eff))) {+ continue+ }+ c := EnvChange{Var: k, Key: k, Value: v}+ switch {+ case isGeneratedSecret(k):+ c.Secret = k != secrets.VAPIDPublicKey+ if _, chosen := s.Settings[k]; !chosen {+ c.Kind = ChangeSecret+ }+ case settings.Twins[k] != "":+ c.Kind, c.Key = ChangeAdmin, settings.Twins[k]+ if translated, ok := admin[c.Key]; ok {+ c.Value = translated+ }+ c.Secret = settings.SecretName(k)+ case proxies[strings.ToUpper(k)] != "":+ c.Key = proxies[strings.ToUpper(k)]+ c.Secret = hasUserinfo(v)+ case named[k].Name != "":+ c.Key, c.Secret = named[k].Name, named[k].Secret+ case slices.Contains(read, k):+ c.Secret = settings.SecretName(k) || hasUserinfo(v)+ default:+ unused = append(unused, k)+ continue+ }+ if seen[c.Key] {+ continue+ }+ seen[c.Key] = true+ out = append(out, c)+ }+ return out, unused, nil+}++func hasUserinfo(v string) bool {+ scheme, rest, ok := strings.Cut(v, "://")+ if !ok || scheme == "" {+ return false+ }+ host, _, _ := strings.Cut(rest, "/")+ return strings.Contains(host, "@")+}++func applyChanges(s *State, changes []EnvChange) {+ for _, c := range changes {+ switch c.Kind {+ case ChangeSecret:+ s.Secrets[c.Key] = c.Value+ case ChangeAdmin:+ if s.AdminImport == nil {+ s.AdminImport = &AdminImport{Values: map[string]string{}}+ }+ if s.AdminImport.Values == nil {+ s.AdminImport.Values = map[string]string{}+ }+ s.AdminImport.Values[c.Key] = c.Value+ s.AdminImport.SentAt = ""+ default:+ s.Settings[c.Key] = c.Value+ }+ }+}diff --git a/fluxer_rexctl/internal/instance/import.go b/fluxer_rexctl/internal/instance/import.gonew file mode 100644index 000000000..531b429d7--- /dev/null+++ b/fluxer_rexctl/internal/instance/import.go@@ -0,0 +1,509 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++package instance++import (+ "context"+ "errors"+ "fmt"+ "io/fs"+ "maps"+ "os"+ "path/filepath"+ "slices"+ "strconv"+ "strings"+ "testing/fstest"+ "time"++ "go.yaml.in/yaml/v4"++ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/engine"+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/secrets"+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack"+)++type Source string++const (+ SourceInstaller Source = "installer"+ SourceEjected Source = "ejected"+ SourceManaged Source = "managed"+)++const EjectedEnvFile = "ejected.env"++type ImportReport struct {+ Source Source+ Dir string+ Project string+ PodmanCompose bool+ CustomFiles []string+ KeepAsMine []string+ Mine map[string]string+ DeleteFiles []string+ Minted []string+ Placeholders []string+ Dropped []string+ Defaults []string+ AdminKeys []string+ Changed []string+ DedupedEnv bool+ EngineProxies bool+ RemovedEnv bool+}++func (r *ImportReport) DroppedLine() string {+ if len(r.Dropped) == 0 {+ return ""+ }+ saved := "They are"+ if len(r.Dropped) == 1 {+ saved = "It's"+ }+ return fmt.Sprintf("rexctl left out %s Fluxer doesn't read: %s. %s saved in %s.",+ count(len(r.Dropped), "setting", "settings"), strings.Join(r.Dropped, ", "), saved,+ filepath.Join(StateDirName, ReleasesDir, LegacyRelease, EnvFileName))+}++func count(n int, one, many string) string {+ if n == 1 {+ return "1 " + one+ }+ return strconv.Itoa(n) + " " + many+}++func Import(dir string, e *engine.Engine) (*State, *ImportReport, error) {+ return runImport(context.Background(), dir, e, false)+}++func PreviewImport(dir string, e *engine.Engine) (*ImportReport, error) {+ _, r, err := runImport(context.Background(), dir, e, true)+ return r, err+}++func runImport(ctx context.Context, dir string, e *engine.Engine, dry bool) (*State, *ImportReport, error) {+ abs, err := filepath.Abs(dir)+ if err != nil {+ return nil, nil, err+ }+ statePath := filepath.Join(abs, StateDirName, StateFileName)+ data, err := os.ReadFile(statePath)+ if errors.Is(err, fs.ErrNotExist) {+ return importInstaller(ctx, abs, e, dry)+ }+ if err != nil {+ return nil, nil, err+ }+ s, err := ParseState(data, abs, filepath.Join(abs, EnvFileName))+ if err != nil {+ return nil, nil, err+ }+ switch {+ case s.MigratedFromV1():+ return importRexctl(ctx, abs, e, s, dry)+ case s.Ejected != nil:+ return importEjected(ctx, abs, e, s, dry)+ }+ r := &ImportReport{Source: SourceManaged, Dir: abs, Project: s.Project}+ inst := &Instance{Dir: abs, State: s}+ if d, err := envDrift(inst); err == nil && d != nil && d.Same && !dry {+ if err := removeIfExists(inst.Path(EnvFileName)); err != nil {+ return nil, nil, err+ }+ r.RemovedEnv = true+ }+ return s, r, nil+}++func refusal(what string) *AdoptError {+ return &AdoptError{+ Msg: "rexctl can't manage this setup.",+ Fix: "Keep running it with docker compose, or " + what + " and run rexctl takeover again.",+ }+}++func importInstaller(ctx context.Context, dir string, e *engine.Engine, dry bool) (*State, *ImportReport, error) {+ l, err := DetectLegacy(dir)+ if err != nil {+ return nil, nil, err+ }+ p, err := PlanAdopt(l)+ if err != nil {+ return nil, nil, err+ }+ r := &ImportReport{+ Source: SourceInstaller,+ Dir: dir,+ Project: p.Project,+ CustomFiles: p.CustomFiles,+ KeepAsMine: p.KeepAsMine,+ Mine: p.Mine,+ DeleteFiles: p.DeleteFiles,+ DedupedEnv: p.DedupedEnv,+ }+ var custom []byte+ if p.customFrom != "" {+ if custom, err = l.read(p.customFrom); err != nil {+ return nil, nil, err+ }+ if what := customProblem(custom); what != "" {+ return nil, nil, refusal(fmt.Sprintf("remove %s from %s", what, displayPath(dir, p.customFrom)))+ }+ }+ live, err := inspect(ctx, e, p.Project, dir)+ if err != nil {+ return nil, nil, err+ }+ r.PodmanCompose = live.podmanCompose+ if live.podmanCompose && e.Kind == engine.Podman && VersionBelow(e.Version, engine.MinPodman) {+ return nil, nil, &AdoptError{+ Msg: "rexctl can't manage this setup.",+ Fix: fmt.Sprintf("Keep running it with podman-compose, or upgrade Podman to %s or newer and run rexctl takeover again.", engine.MinPodman),+ }+ }+ set, err := legacySet(l)+ if err != nil {+ return nil, nil, err+ }+ examples := map[string]string{}+ if data, err := l.read(filepath.Join(dir, ".env.example")); err == nil {+ if m, _, err := parseEnv(data); err == nil {+ examples = m+ }+ }+ envData, err := l.read(filepath.Join(dir, EnvFileName))+ if err != nil {+ return nil, nil, err+ }+ env, _, err := parseEnv(envData)+ if err != nil {+ return nil, nil, err+ }+ overlays := make([]stack.Overlay, 0, len(p.Overlays))+ for _, o := range p.Overlays {+ overlays = append(overlays, stack.Overlay(o))+ }+ vals, err := importValues(valueInput{+ env: env, set: set, custom: custom, overlays: overlays, legacy: true,+ examples: examples, proxies: live.proxies,+ })+ if err != nil {+ return nil, nil, err+ }+ minted, err := mint(vals.secrets, live.haveVolumes)+ if err != nil {+ return nil, nil, err+ }+ fill(r, vals, minted)+ if dry {+ return nil, r, nil+ }+ if err := adoptFiles(ctx, p); err != nil {+ return nil, nil, err+ }+ installed := time.Now().UTC().Truncate(time.Second)+ if fi, err := os.Stat(filepath.Join(l.snapshotDir(), EnvFileName)); err == nil {+ installed = fi.ModTime().UTC().Truncate(time.Second)+ }+ s := &State{+ Schema: StateSchema,+ Project: p.Project,+ Release: LegacyRelease,+ Images: Images{Release: LegacyRelease, Refs: live.images},+ Settings: vals.settings,+ Secrets: vals.secrets,+ SetupDone: true,+ InstalledAt: installed,+ UpdatedAt: time.Now().UTC().Truncate(time.Second),+ }+ if len(vals.admin) > 0 {+ s.AdminImport = &AdminImport{Values: vals.admin}+ }+ pin(s, e)+ s.normalize()+ if err := finishImport(dir, s); err != nil {+ return nil, nil, err+ }+ return s, r, nil+}++func finishImport(dir string, s *State) error {+ if err := writeReleaseSet(dir, s.Release); err != nil {+ return err+ }+ if err := recordRendered(dir, s); err != nil {+ return err+ }+ s.UpdatedAt = time.Now().UTC().Truncate(time.Second)+ if err := s.Save(filepath.Join(dir, StateDirName, StateFileName)); err != nil {+ return err+ }+ return removeIfExists(filepath.Join(dir, EnvFileName))+}++func fill(r *ImportReport, vals *valueResult, minted []string) {+ r.Minted = minted+ r.Placeholders = vals.placeholders+ r.Dropped = vals.dropped+ r.Defaults = vals.defaults+ r.AdminKeys = slices.Sorted(maps.Keys(vals.admin))+ r.EngineProxies = vals.engineProxies+}++func pin(s *State, e *engine.Engine) {+ if e == nil {+ return+ }+ s.Engine = EngineRef{Kind: string(e.Kind), Host: e.Host, ID: e.ID}+}++func mint(values map[string]string, haveVolumes bool) ([]string, error) {+ minted, err := secrets.Mint(values, haveVolumes)+ var missing *secrets.StatefulMissingError+ if errors.As(err, &missing) {+ return nil, &AdoptError{+ Msg: fmt.Sprintf("Your instance's data needs %s, and rexctl can't find %s.", strings.Join(missing.Keys, ", "), pick(len(missing.Keys), "its value", "their values")),+ Fix: "Restore the instance from a backup with rexctl restore.",+ }+ }+ return minted, err+}++func pick(n int, one, many string) string {+ if n == 1 {+ return one+ }+ return many+}++type liveState struct {+ haveVolumes bool+ podmanCompose bool+ images map[string]string+ proxies engine.Proxies+}++func inspect(ctx context.Context, e *engine.Engine, project, dir string) (*liveState, error) {+ live := &liveState{haveVolumes: true, images: map[string]string{}}+ if e == nil {+ return live, nil+ }+ live.proxies = e.Proxies()+ vols, err := e.Volumes(ctx, project)+ if err != nil {+ return nil, err+ }+ live.haveVolumes = len(vols) > 0+ containers, err := e.Named(project, dir).PS(ctx)+ if err != nil {+ return nil, err+ }+ for _, c := range containers {+ if c.Labels[engine.PodmanComposeLabel] != "" {+ live.podmanCompose = true+ }+ if c.Service == "" || c.Image == "" {+ continue+ }+ live.images[c.Service] = c.Image+ if strings.Contains(c.Image, "@") || c.ImageID == "" {+ continue+ }+ if d, err := e.RepoDigest(ctx, c.ImageID, c.Image); err == nil && d != "" {+ live.images[c.Service] = d+ }+ }+ return live, nil+}++func VersionBelow(version, floor string) bool {+ parse := func(v string) []int {+ v = strings.TrimPrefix(strings.TrimSpace(v), "v")+ v, _, _ = strings.Cut(v, "-")+ var out []int+ for _, part := range strings.SplitN(v, ".", 3) {+ n, err := strconv.Atoi(part)+ if err != nil {+ return nil+ }+ out = append(out, n)+ }+ return out+ }+ a, b := parse(version), parse(floor)+ if a == nil || b == nil {+ return false+ }+ for i := range b {+ x := 0+ if i < len(a) {+ x = a[i]+ }+ if x != b[i] {+ return x < b[i]+ }+ }+ return false+}++func customProblem(data []byte) string {+ var doc map[string]any+ if err := yaml.Unmarshal(data, &doc); err != nil {+ return ""+ }+ if _, ok := doc["include"]; ok {+ return "include"+ }+ services, _ := doc["services"].(map[string]any)+ for _, name := range slices.Sorted(maps.Keys(services)) {+ svc, _ := services[name].(map[string]any)+ if _, ok := svc["build"]; ok {+ return "build"+ }+ if _, ok := svc["env_file"]; ok {+ return "env_file"+ }+ switch ext := svc["extends"].(type) {+ case map[string]any:+ if _, ok := ext["file"]; ok {+ return "extends"+ }+ }+ }+ return ""+}++func legacySet(l *Legacy) (*stack.Set, error) {+ files := fstest.MapFS{}+ for i, f := range l.ComposeFiles {+ data, err := l.read(f)+ if err != nil {+ continue+ }+ name := filepath.Base(f)+ if i == 0 {+ name = stack.ComposeFile+ } else if overlayFor(f, l.Dir) == "" {+ continue+ }+ files[name] = &fstest.MapFile{Data: normalizeLF(data), Mode: 0o444}+ }+ if data, err := l.read(filepath.Join(l.Dir, stack.Caddyfile)); err == nil {+ files[stack.Caddyfile] = &fstest.MapFile{Data: normalizeLF(data), Mode: 0o444}+ }+ return newSet(LegacyRelease, files, nil)+}++func newSet(release string, files fs.FS, profiles map[string]map[string]string) (*stack.Set, error) {+ vars, err := stack.FilesVars(files)+ if err != nil {+ return nil, err+ }+ if profiles == nil {+ profiles = map[string]map[string]string{}+ }+ return &stack.Set{Release: release, Files: files, Profiles: profiles, Vars: vars}, nil+}++func renderedNames(dir string) []string {+ names := []string{stack.ComposeFile}+ for _, o := range slices.Sorted(maps.Keys(stack.OverlayFiles)) {+ names = append(names, stack.OverlayFiles[o])+ }+ names = append(names, stack.Caddyfile)+ var out []string+ for _, n := range names {+ if isFile(filepath.Join(dir, n)) {+ out = append(out, n)+ }+ }+ return out+}++func diskSet(dir, release string, profiles map[string]map[string]string) (*stack.Set, error) {+ files := fstest.MapFS{}+ for _, name := range renderedNames(dir) {+ data, err := os.ReadFile(filepath.Join(dir, name))+ if err != nil {+ return nil, err+ }+ files[name] = &fstest.MapFile{Data: data, Mode: 0o444}+ }+ if _, ok := files[stack.ComposeFile]; !ok {+ return nil, fmt.Errorf("%w: %s", ErrNoReleaseSet, release)+ }+ return newSet(release, files, profiles)+}++func writeReleaseSet(dir, release string) error {+ target := filepath.Join(dir, StateDirName, ReleasesDir, release, StackDir)+ if isDir(target) {+ return nil+ }+ var set *stack.Set+ if embedded := stack.Embedded(); release == embedded.Release {+ set = embedded+ } else {+ var profiles map[string]map[string]string+ if release != LegacyRelease {+ profiles = stack.Embedded().Profiles+ }+ var err error+ if set, err = diskSet(dir, release, profiles); err != nil {+ return err+ }+ }+ return stack.WriteSet(target, set)+}++func recordRendered(dir string, s *State) error {+ if len(s.Rendered) > 0 {+ return nil+ }+ if s.Rendered == nil {+ s.Rendered = map[string]string{}+ }+ for _, name := range renderedNames(dir) {+ data, err := os.ReadFile(filepath.Join(dir, name))+ if err != nil {+ return err+ }+ s.Rendered[name] = stack.Hash(data)+ }+ return nil+}++func releaseSet(dir, release string) (*stack.Set, error) {+ set, err := stack.LoadSet(filepath.Join(dir, StateDirName, ReleasesDir, release, StackDir))+ if err == nil {+ return set, nil+ }+ if embedded := stack.Embedded(); release == embedded.Release {+ return embedded, nil+ }+ var profiles map[string]map[string]string+ if release != LegacyRelease {+ profiles = stack.Embedded().Profiles+ }+ return diskSet(filesDir, release, profiles)+}++func readCustom(dir string) []byte {+ data, _ := os.ReadFile(filepath.Join(dir, CustomCompose))+ return data+}++func otherEngine(pinned, connected string) *AdoptError {+ return refusal(fmt.Sprintf("connect rexctl to %s instead of %s", engineName(pinned), engineName(connected)))+}++func engineName(kind string) string {+ switch kind {+ case string(engine.Podman):+ return "Podman"+ case string(engine.Docker):+ return "Docker"+ }+ return "the engine that runs it"+}