Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
16.9 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

package instance

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/fs"
	"maps"
	"os"
	"path/filepath"
	"reflect"
	"slices"
	"strconv"
	"strings"
	"testing/fstest"
	"time"

	"go.yaml.in/yaml/v4"

	"github.com/fluxerapp/fluxer/fluxer_rexctl/internal/engine"
	"github.com/fluxerapp/fluxer/fluxer_rexctl/internal/envfile"
	"github.com/fluxerapp/fluxer/fluxer_rexctl/internal/secrets"
	"github.com/fluxerapp/fluxer/fluxer_rexctl/internal/settings"
	"github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack"
)

type Source string

const (
	SourceInstaller Source = "installer"
	SourceEjected   Source = "ejected"
	SourceManaged   Source = "managed"
)

const EjectedEnvFile = "ejected.env"

type ImportReport struct {
	Source        Source
	Dir           string
	Project       string
	PodmanCompose bool
	CustomFiles   []string
	KeepAsMine    []string
	Mine          map[string]string
	DeleteFiles   []string
	Minted        []string
	Placeholders  []string
	Dropped       []string
	Defaults      []string
	AdminKeys     []string
	Changed       []string
	DedupedEnv    bool
	EngineProxies bool
	ListenAddress string
	RemovedEnv    bool
	Images        map[string]string
	Release       string
}

func (r *ImportReport) DroppedLine() string {
	if len(r.Dropped) == 0 {
		return ""
	}
	saved := "They are"
	if len(r.Dropped) == 1 {
		saved = "It's"
	}
	return fmt.Sprintf("rexctl left out %s Fluxer doesn't read: %s. %s saved in %s.",
		count(len(r.Dropped), "setting", "settings"), strings.Join(r.Dropped, ", "), saved,
		filepath.Join(StateDirName, ReleasesDir, LegacyRelease, EnvFileName))
}

func count(n int, one, many string) string {
	if n == 1 {
		return "1 " + one
	}
	return strconv.Itoa(n) + " " + many
}

func Import(dir string, e *engine.Engine) (*State, *ImportReport, error) {
	return runImport(context.Background(), dir, e, false)
}

func PreviewImport(dir string, e *engine.Engine) (*ImportReport, error) {
	_, r, err := runImport(context.Background(), dir, e, true)
	return r, err
}

func runImport(ctx context.Context, dir string, e *engine.Engine, dry bool) (*State, *ImportReport, error) {
	abs, err := filepath.Abs(dir)
	if err != nil {
		return nil, nil, err
	}
	statePath := filepath.Join(abs, StateDirName, StateFileName)
	data, err := os.ReadFile(statePath)
	if errors.Is(err, fs.ErrNotExist) {
		return importInstaller(ctx, abs, e, dry)
	}
	if err != nil {
		return nil, nil, err
	}
	s, err := ParseState(data, abs)
	if err != nil {
		return nil, nil, err
	}
	if s.Ejected != nil {
		return importEjected(ctx, abs, e, s, dry)
	}
	r := &ImportReport{Source: SourceManaged, Dir: abs, Project: s.Project}
	inst := &Instance{Dir: abs, State: s}
	if d, err := envDrift(inst); err == nil && d != nil && d.Same && !dry {
		if err := removeIfExists(inst.Path(EnvFileName)); err != nil {
			return nil, nil, err
		}
		r.RemovedEnv = true
	}
	return s, r, nil
}

func refusal(what string) *AdoptError {
	return &AdoptError{
		Msg: "rexctl can't manage this setup.",
		Fix: "Keep running it with docker compose, or " + what + " and run rexctl takeover again.",
	}
}

func importInstaller(ctx context.Context, dir string, e *engine.Engine, dry bool) (*State, *ImportReport, error) {
	l, err := DetectLegacy(dir)
	if err != nil {
		return nil, nil, err
	}
	p, err := PlanAdopt(l)
	if err != nil {
		return nil, nil, err
	}
	r := &ImportReport{
		Source:      SourceInstaller,
		Dir:         dir,
		Project:     p.Project,
		CustomFiles: p.CustomFiles,
		KeepAsMine:  p.KeepAsMine,
		Mine:        p.Mine,
		DeleteFiles: p.DeleteFiles,
		DedupedEnv:  p.DedupedEnv,
	}
	var custom []byte
	if p.customFrom != "" {
		if custom, err = l.read(p.customFrom); err != nil {
			return nil, nil, err
		}
		if what := customProblem(custom); what != "" {
			return nil, nil, refusal(fmt.Sprintf("remove %s from %s", what, displayPath(dir, p.customFrom)))
		}
	}
	live, err := inspect(ctx, e, p.Project, dir)
	if err != nil {
		return nil, nil, err
	}
	r.PodmanCompose = live.podmanCompose
	r.Images = live.images
	if live.podmanCompose && e.Kind == engine.Podman && VersionBelow(e.Version, engine.MinPodman) {
		return nil, nil, &AdoptError{
			Msg: "rexctl can't manage this setup.",
			Fix: fmt.Sprintf("Keep running it with podman-compose, or upgrade Podman to %s or newer and run rexctl takeover again.", engine.MinPodman),
		}
	}
	set, err := legacySet(l)
	if err != nil {
		return nil, nil, err
	}
	examples := map[string]string{}
	if data, err := l.read(filepath.Join(dir, ".env.example")); err == nil {
		if m, _, err := parseEnv(data); err == nil {
			examples = m
		}
	}
	envData, err := l.read(filepath.Join(dir, EnvFileName))
	if err != nil {
		return nil, nil, err
	}
	env, _, err := parseEnv(envData)
	if err != nil {
		return nil, nil, err
	}
	overlays := make([]stack.Overlay, 0, len(p.Overlays))
	for _, o := range p.Overlays {
		overlays = append(overlays, stack.Overlay(o))
	}
	vals, err := importValues(valueInput{
		env: env, set: set, custom: custom, overlays: overlays,
		examples: examples, proxies: live.proxies,
	})
	if err != nil {
		return nil, nil, err
	}
	minted, err := mint(vals.secrets, live.haveVolumes)
	if err != nil {
		return nil, nil, err
	}
	fill(r, vals, minted)
	if dry {
		return nil, r, nil
	}
	if err := adoptFiles(ctx, p); err != nil {
		return nil, nil, err
	}
	installed := time.Now().UTC().Truncate(time.Second)
	if fi, err := os.Stat(filepath.Join(l.snapshotDir(), EnvFileName)); err == nil {
		installed = fi.ModTime().UTC().Truncate(time.Second)
	}
	release, refs := LegacyRelease, live.images
	if images, ok := bundledRelease(l, p, custom); ok {
		release, refs = images.Release, images.Images
	}
	s := &State{
		Schema:      StateSchema,
		Project:     p.Project,
		Release:     release,
		Images:      Images{Release: release, Refs: refs},
		Settings:    vals.settings,
		Secrets:     vals.secrets,
		SetupDone:   true,
		InstalledAt: installed,
		UpdatedAt:   time.Now().UTC().Truncate(time.Second),
	}
	if len(vals.admin) > 0 {
		s.AdminImport = &AdminImport{Values: vals.admin}
	}
	if release != LegacyRelease {
		ConvertLegacy(s, stack.Embedded(), refs)
	}
	pin(s, e)
	s.normalize()
	if err := finishImport(dir, s); err != nil {
		return nil, nil, err
	}
	return s, r, nil
}

func bundledRelease(l *Legacy, p *AdoptPlan, custom []byte) (stack.ImageSet, bool) {
	images, err := stack.EmbeddedImages()
	if err != nil || len(images.Images) == 0 || images.Release != stack.Embedded().Release {
		return images, false
	}
	if slices.ContainsFunc(p.KeepAsMine, func(name string) bool { return name != ".env.example" }) || setsImage(custom, images.Images) {
		return images, false
	}
	data, err := l.read(l.ComposeFiles[0])
	if err != nil {
		return images, false
	}
	want, err := stack.RenderCompose(stack.Embedded(), images.Images, p.Project)
	return images, err == nil && bytes.Equal(normalizeLF(data), want)
}

func setsImage(custom []byte, images map[string]string) bool {
	var doc struct {
		Services map[string]map[string]any `yaml:"services"`
	}
	if err := yaml.Unmarshal(custom, &doc); err != nil {
		return true
	}
	fluxer := stack.RegistryServices()
	for name, svc := range doc.Services {
		if _, ok := svc["image"]; ok && fluxer[name] && images[name] != "" {
			return true
		}
	}
	return false
}

func finishImport(dir string, s *State) error {
	if err := writeReleaseSet(dir, s.Release); err != nil {
		return err
	}
	if err := recordRendered(dir, s); err != nil {
		return err
	}
	s.UpdatedAt = time.Now().UTC().Truncate(time.Second)
	s.rootLayout = true
	if err := s.Save(filepath.Join(dir, StateDirName, StateFileName)); err != nil {
		return err
	}
	return removeIfExists(filepath.Join(dir, EnvFileName))
}

func fill(r *ImportReport, vals *valueResult, minted []string) {
	r.Minted = minted
	r.Placeholders = vals.placeholders
	r.Dropped = vals.dropped
	r.Defaults = vals.defaults
	r.AdminKeys = slices.Sorted(maps.Keys(vals.admin))
	r.EngineProxies = vals.engineProxies
	r.ListenAddress = vals.settings[settings.EdgeListen]
}

func pin(s *State, e *engine.Engine) {
	if e == nil {
		return
	}
	s.Engine = EngineRef{Kind: string(e.Kind), Host: e.Host, ID: e.ID}
}

func mint(values map[string]string, haveVolumes bool) ([]string, error) {
	minted, err := secrets.Mint(values, haveVolumes)
	var missing *secrets.StatefulMissingError
	if errors.As(err, &missing) {
		return nil, &AdoptError{
			Msg: fmt.Sprintf("Your instance's data needs %s, and rexctl can't find %s.", strings.Join(missing.Keys, ", "), pick(len(missing.Keys), "its value", "their values")),
			Fix: "Restore the instance from a backup with rexctl restore.",
		}
	}
	return minted, err
}

func pick(n int, one, many string) string {
	if n == 1 {
		return one
	}
	return many
}

type liveState struct {
	haveVolumes   bool
	podmanCompose bool
	images        map[string]string
	proxies       engine.Proxies
}

func inspect(ctx context.Context, e *engine.Engine, project, dir string) (*liveState, error) {
	live := &liveState{haveVolumes: true, images: map[string]string{}}
	if e == nil {
		return live, nil
	}
	live.proxies = e.Proxies()
	vols, err := e.Volumes(ctx, project)
	if err != nil {
		return nil, err
	}
	live.haveVolumes = len(vols) > 0
	containers, err := e.Named(project, dir).PS(ctx)
	if err != nil {
		return nil, err
	}
	for _, c := range containers {
		if c.Labels[engine.PodmanComposeLabel] != "" {
			live.podmanCompose = true
		}
		if c.Service == "" || c.Image == "" {
			continue
		}
		live.images[c.Service] = c.Image
		if strings.Contains(c.Image, "@") || c.ImageID == "" {
			continue
		}
		if d, err := e.RepoDigest(ctx, c.ImageID, c.Image); err == nil && d != "" {
			live.images[c.Service] = d
		}
	}
	return live, nil
}

func VersionBelow(version, floor string) bool {
	parse := func(v string) []int {
		v = strings.TrimPrefix(strings.TrimSpace(v), "v")
		v, _, _ = strings.Cut(v, "-")
		var out []int
		for _, part := range strings.SplitN(v, ".", 3) {
			n, err := strconv.Atoi(part)
			if err != nil {
				return nil
			}
			out = append(out, n)
		}
		return out
	}
	a, b := parse(version), parse(floor)
	if a == nil || b == nil {
		return false
	}
	for i := range b {
		x := 0
		if i < len(a) {
			x = a[i]
		}
		if x != b[i] {
			return x < b[i]
		}
	}
	return false
}

func customProblem(data []byte) string {
	var doc map[string]any
	if err := yaml.Unmarshal(data, &doc); err != nil {
		return ""
	}
	if _, ok := doc["include"]; ok {
		return "include"
	}
	services, _ := doc["services"].(map[string]any)
	for _, name := range slices.Sorted(maps.Keys(services)) {
		svc, _ := services[name].(map[string]any)
		if _, ok := svc["build"]; ok {
			return "build"
		}
		if _, ok := svc["env_file"]; ok {
			return "env_file"
		}
		switch ext := svc["extends"].(type) {
		case map[string]any:
			if _, ok := ext["file"]; ok {
				return "extends"
			}
		}
	}
	return ""
}

func legacySet(l *Legacy) (*stack.Set, error) {
	files := fstest.MapFS{}
	for i, f := range l.ComposeFiles {
		data, err := l.read(f)
		if err != nil {
			continue
		}
		name := filepath.Base(f)
		if i == 0 {
			name = stack.ComposeFile
		} else if overlayFor(f, l.Dir) == "" {
			continue
		}
		files[name] = &fstest.MapFile{Data: normalizeLF(data), Mode: 0o444}
	}
	if data, err := l.read(filepath.Join(l.Dir, stack.Caddyfile)); err == nil {
		files[stack.Caddyfile] = &fstest.MapFile{Data: normalizeLF(data), Mode: 0o444}
	}
	return newSet(LegacyRelease, files, nil)
}

func newSet(release string, files fs.FS, profiles map[string]map[string]string) (*stack.Set, error) {
	vars, err := stack.FilesVars(files)
	if err != nil {
		return nil, err
	}
	if profiles == nil {
		profiles = map[string]map[string]string{}
	}
	return &stack.Set{Release: release, Files: files, Profiles: profiles, Vars: vars}, nil
}

func renderedNames(dir string) []string {
	var out []string
	for _, n := range stack.FileNames() {
		if isFile(filepath.Join(dir, n)) {
			out = append(out, n)
		}
	}
	return out
}

func diskSet(dir, release string, profiles map[string]map[string]string) (*stack.Set, error) {
	files := fstest.MapFS{}
	for _, name := range renderedNames(dir) {
		data, err := os.ReadFile(filepath.Join(dir, name))
		if err != nil {
			return nil, err
		}
		files[name] = &fstest.MapFile{Data: data, Mode: 0o444}
	}
	if _, ok := files[stack.ComposeFile]; !ok {
		return nil, fmt.Errorf("%w: %s", ErrNoReleaseSet, release)
	}
	return newSet(release, files, profiles)
}

func validRelease(release string) bool {
	return release == "" || (filepath.IsLocal(release) && release == filepath.Base(release))
}

func writeReleaseSet(dir, release string) error {
	if !validRelease(release) {
		return fmt.Errorf("%w: %s", ErrNoReleaseSet, release)
	}
	target := filepath.Join(dir, StateDirName, ReleasesDir, release, StackDir)
	if isDir(target) {
		return nil
	}
	var set *stack.Set
	if embedded := stack.Embedded(); release == embedded.Release {
		set = embedded
	} else {
		var profiles map[string]map[string]string
		if release != LegacyRelease {
			profiles = stack.Embedded().Profiles
		}
		var err error
		if set, err = diskSet(dir, release, profiles); err != nil {
			return err
		}
	}
	return stack.WriteSet(target, set)
}

func recordRendered(dir string, s *State) error {
	if len(s.Rendered) > 0 {
		return nil
	}
	if s.Rendered == nil {
		s.Rendered = map[string]string{}
	}
	for _, name := range renderedNames(dir) {
		data, err := os.ReadFile(filepath.Join(dir, name))
		if err != nil {
			return err
		}
		s.Rendered[name] = stack.Hash(data)
	}
	return nil
}

func releaseSet(dir, release string) (*stack.Set, error) {
	if !validRelease(release) {
		return nil, fmt.Errorf("%w: %s", ErrNoReleaseSet, release)
	}
	set, err := stack.LoadSet(filepath.Join(dir, StateDirName, ReleasesDir, release, StackDir))
	if err == nil {
		return set, nil
	}
	if embedded := stack.Embedded(); release == embedded.Release {
		return embedded, nil
	}
	var profiles map[string]map[string]string
	if release != LegacyRelease {
		profiles = stack.Embedded().Profiles
	}
	return diskSet(dir, release, profiles)
}

func RetiredService(dir string) string {
	var doc map[string]any
	if err := yaml.Unmarshal(readCustom(dir), &doc); err != nil {
		return ""
	}
	services, _ := doc["services"].(map[string]any)
	for _, name := range slices.Sorted(maps.Keys(services)) {
		svc, _ := services[name].(map[string]any)
		if _, ok := svc["image"]; ok || slices.Contains(stack.Services, name) {
			continue
		}
		return name
	}
	return ""
}

func DropMergedShards(dir string) ([]string, error) {
	path := filepath.Join(dir, CustomCompose)
	data, err := os.ReadFile(path)
	if errors.Is(err, fs.ErrNotExist) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	out, dropped := withoutMergedShards(data)
	if len(dropped) == 0 {
		return nil, nil
	}
	return dropped, envfile.WriteFileAtomic(path, out, false)
}

func withoutMergedShards(data []byte) ([]byte, []string) {
	var doc yaml.Node
	var before map[string]any
	if yaml.Unmarshal(data, &doc) != nil || yaml.Unmarshal(data, &before) != nil || len(doc.Content) == 0 {
		return nil, nil
	}
	root := doc.Content[0]
	if root.Kind != yaml.MappingNode {
		return nil, nil
	}
	var services *yaml.Node
	for i := 0; i+1 < len(root.Content); i += 2 {
		if root.Content[i].Value == "services" {
			services = root.Content[i+1]
		}
	}
	decoded, _ := before["services"].(map[string]any)
	if services == nil || services.Kind != yaml.MappingNode || services.Style&yaml.FlowStyle != 0 || decoded == nil {
		return nil, nil
	}
	lines := strings.Split(string(data), "\n")
	remove := make([]bool, len(lines))
	var dropped []string
	for i := 0; i+1 < len(services.Content); i += 2 {
		key := services.Content[i]
		base, ok := strings.CutSuffix(key.Value, "-shard")
		if !ok || !slices.Contains(stack.Services, base) || slices.Contains(stack.Services, key.Value) {
			continue
		}
		shard, have := decoded[key.Value]
		if want, ok := decoded[base]; !have || !ok || !reflect.DeepEqual(shard, want) {
			continue
		}
		start, end := key.Line-1, key.Line
		for end < len(lines) && (strings.TrimSpace(lines[end]) == "" || indent(lines[end]) >= key.Column) {
			end++
		}
		for end > start+1 && strings.TrimSpace(lines[end-1]) == "" {
			end--
		}
		for j := start; j < end; j++ {
			remove[j] = true
		}
		dropped = append(dropped, key.Value)
	}
	if len(dropped) == 0 {
		return nil, nil
	}
	kept := make([]string, 0, len(lines))
	for i, l := range lines {
		if !remove[i] {
			kept = append(kept, l)
		}
	}
	out := []byte(strings.Join(kept, "\n"))
	var after map[string]any
	if yaml.Unmarshal(out, &after) != nil {
		return nil, nil
	}
	for _, name := range dropped {
		delete(decoded, name)
	}
	if !reflect.DeepEqual(before, after) {
		return nil, nil
	}
	slices.Sort(dropped)
	return out, dropped
}

func indent(line string) int {
	return len(line) - len(strings.TrimLeft(line, " "))
}

func readCustom(dir string) []byte {
	data, _ := os.ReadFile(filepath.Join(dir, CustomCompose))
	return data
}

func otherEngine(pinned, connected string) *AdoptError {
	return refusal(fmt.Sprintf("connect rexctl to %s instead of %s", engineName(pinned), engineName(connected)))
}

func engineName(kind string) string {
	switch kind {
	case string(engine.Podman):
		return "Podman"
	case string(engine.Docker):
		return "Docker"
	}
	return "the engine that runs it"
}