A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(rexctl): size services from memory and cpus on every start
Hampusdb4cbe501027
diff --git a/fluxer_docs/src/content/docs/operator/rexctl.mdx b/fluxer_docs/src/content/docs/operator/rexctl.mdxindex 4e9969a27..6ccc95878 100644--- a/fluxer_docs/src/content/docs/operator/rexctl.mdx+++ b/fluxer_docs/src/content/docs/operator/rexctl.mdx@@ -290,7 +290,7 @@ rexctl install [flags]- `--memory-profile string`- Small, medium or large (picked from this machine's memory).+ Pin small, medium or large instead of sizing Fluxer to this machine on every start.- `--name string`diff --git a/fluxer_docs/src/content/docs/operator/running.mdx b/fluxer_docs/src/content/docs/operator/running.mdxindex b27b8bf67..c3abe270e 100644--- a/fluxer_docs/src/content/docs/operator/running.mdx+++ b/fluxer_docs/src/content/docs/operator/running.mdx@@ -107,6 +107,8 @@ This lists every setting with its value and a line of help, and says which settirexctl config set <key> <value>```+Each time rexctl starts Fluxer, it sizes memory limits, database connections and thread counts to the machine's memory and CPUs, so a resized machine needs no changes. The `memory.profile` setting pins a fixed size instead. An instance installed by an older rexctl keeps the size it picked at install. Run `rexctl config unset memory.profile` to let rexctl size it.+## When something is wrong```shdiff --git a/fluxer_rexctl/cmd/install.go b/fluxer_rexctl/cmd/install.goindex b2a3c2622..dfd3d672f 100644--- a/fluxer_rexctl/cmd/install.go+++ b/fluxer_rexctl/cmd/install.go@@ -88,7 +88,7 @@ func newInstallCmd(app *App) *cobra.Command {fl.BoolVar(&f.BehindProxy, "behind-proxy", false, "serve plain HTTP for your own reverse proxy")fl.StringVar(&f.Bind, "bind", "127.0.0.1:8080", "address your proxy connects to, with --behind-proxy")fl.StringVar(&f.VoiceIP, "voice-ip", "", "public IP address for voice and video (found automatically)")- fl.StringVar(&f.MemoryProfile, "memory-profile", "", "small, medium or large (picked from this machine's memory)")+ fl.StringVar(&f.MemoryProfile, "memory-profile", "", "pin small, medium or large instead of sizing Fluxer to this machine on every start")fl.BoolVar(&f.InstallDocker, "install-docker", true, "install Docker if it's missing")fl.BoolVar(&f.OpenFirewall, "open-firewall", true, "open the ports Fluxer needs in ufw or firewalld")fl.BoolVar(&f.SetupPodman, "setup-podman", true, "start Podman's API socket and start Fluxer with the machine")@@ -856,8 +856,6 @@ func (in *installer) write(ctx context.Context, inst *instance.Instance, deleteO}if f.MemoryProfile != "" {st.Settings[settings.MemoryProfile] = f.MemoryProfile- } else if in.host != nil && in.host.MemoryBytes > 0 {- st.Settings[settings.MemoryProfile] = string(stack.ProfileFor(in.host.MemoryBytes))}if _, err := secrets.Mint(st.Secrets, false); err != nil {return ui.Wrap(err, "Couldn't create your instance's secrets.", "Run rexctl install again.")diff --git a/fluxer_rexctl/internal/deploy/set.go b/fluxer_rexctl/internal/deploy/set.goindex 165e1d549..1177b3b74 100644--- a/fluxer_rexctl/internal/deploy/set.go+++ b/fluxer_rexctl/internal/deploy/set.go@@ -37,7 +37,7 @@ func Promised(set *stack.Set, custom []byte, k settings.Key) []string {case settings.EdgeMode, settings.IPv6, settings.ImagesRegistry, settings.StorageEndpoint:return slices.Clone(stack.Services)case settings.MemoryProfile:- return settings.Services(set, custom, stack.ProfileKeys(set))+ return settings.Services(set, custom, append(stack.ProfileKeys(set), stack.SizingKeys()...))}return settings.Services(set, custom, append(slices.Clone(k.Vars), derivedVars[k.Name]...))}diff --git a/fluxer_rexctl/internal/host/cgroup_linux.go b/fluxer_rexctl/internal/host/cgroup_linux.gonew file mode 100644index 000000000..1f2fa9b2e--- /dev/null+++ b/fluxer_rexctl/internal/host/cgroup_linux.go@@ -0,0 +1,69 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++package host++import (+ "os"+ "path/filepath"+ "strconv"+ "strings"+)++const cgroupRoot = "/sys/fs/cgroup"++func cgroupLimits() (memoryBytes uint64, cpus int) {+ data, err := os.ReadFile("/proc/self/cgroup")+ if err != nil {+ return 0, 0+ }+ var own string+ for line := range strings.Lines(string(data)) {+ if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "0::"); ok {+ own = rest+ break+ }+ }+ if own == "" {+ return 0, 0+ }+ dir := filepath.Join(cgroupRoot, own)+ for strings.HasPrefix(dir, cgroupRoot) {+ if v, ok := cgroupMemoryMax(filepath.Join(dir, "memory.max")); ok && (memoryBytes == 0 || v < memoryBytes) {+ memoryBytes = v+ }+ if n := cgroupCPUMax(filepath.Join(dir, "cpu.max")); n > 0 && (cpus == 0 || n < cpus) {+ cpus = n+ }+ if dir == cgroupRoot {+ break+ }+ dir = filepath.Dir(dir)+ }+ return memoryBytes, cpus+}++func cgroupMemoryMax(path string) (uint64, bool) {+ data, err := os.ReadFile(path)+ if err != nil {+ return 0, false+ }+ v, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)+ return v, err == nil && v > 0+}++func cgroupCPUMax(path string) int {+ data, err := os.ReadFile(path)+ if err != nil {+ return 0+ }+ f := strings.Fields(string(data))+ if len(f) != 2 {+ return 0+ }+ quota, err1 := strconv.ParseUint(f[0], 10, 64)+ period, err2 := strconv.ParseUint(f[1], 10, 64)+ if err1 != nil || err2 != nil || period == 0 {+ return 0+ }+ return int((quota + period - 1) / period)+}diff --git a/fluxer_rexctl/internal/host/cgroup_other.go b/fluxer_rexctl/internal/host/cgroup_other.gonew file mode 100644index 000000000..cfc761c7a--- /dev/null+++ b/fluxer_rexctl/internal/host/cgroup_other.go@@ -0,0 +1,7 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++//go:build !linux++package host++func cgroupLimits() (memoryBytes uint64, cpus int) { return 0, 0 }diff --git a/fluxer_rexctl/internal/host/hardware.go b/fluxer_rexctl/internal/host/hardware.gonew file mode 100644index 000000000..3849c1c5e--- /dev/null+++ b/fluxer_rexctl/internal/host/hardware.go@@ -0,0 +1,18 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++package host++import "runtime"++func Hardware() (memoryBytes uint64, cpus int) {+ memoryBytes, _ = Memory()+ cpus = runtime.NumCPU()+ limit, quota := cgroupLimits()+ if limit > 0 && (memoryBytes == 0 || limit < memoryBytes) {+ memoryBytes = limit+ }+ if quota > 0 && quota < cpus {+ cpus = quota+ }+ return memoryBytes, cpus+}diff --git a/fluxer_rexctl/internal/settings/effective.go b/fluxer_rexctl/internal/settings/effective.goindex 2074627e9..19d611601 100644--- a/fluxer_rexctl/internal/settings/effective.go+++ b/fluxer_rexctl/internal/settings/effective.go@@ -8,7 +8,9 @@ import ("slices""strconv""strings"+ "sync"+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/host""github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack")@@ -21,6 +23,12 @@ func Effective(set *stack.Set, chosen, secrets, adminImport map[string]string) (}}+ if len(set.Profiles) > 0 {+ for v, value := range stack.Sizing(Hardware(), chosen[MemoryProfile] == "", fixed) {+ put(v, value)+ }+ }+if name := chosen[MemoryProfile]; name != "" {values, ok := set.Profiles[name]if !ok {@@ -78,6 +86,11 @@ func Effective(set *stack.Set, chosen, secrets, adminImport map[string]string) (return env, nil}+var Hardware = sync.OnceValue(func() stack.Hardware {+ memory, cpus := host.Hardware()+ return stack.Hardware{MemoryBytes: memory, CPUs: cpus}+})+func namedVars(k Key, value string) map[string]string {switch k.Name {case VoicePublicIP:diff --git a/fluxer_rexctl/internal/settings/keys.go b/fluxer_rexctl/internal/settings/keys.goindex 0ff9cea72..f08adab97 100644--- a/fluxer_rexctl/internal/settings/keys.go+++ b/fluxer_rexctl/internal/settings/keys.go@@ -128,7 +128,7 @@ func buildNamed() []Key {{Name: ImagesRegistry, Type: "registry", Validate: validRegistry,Help: "A registry mirror for the database, cache, search, storage, voice and edge images"},{Name: MemoryProfile, Type: "choice", Options: []string{"small", "medium", "large"}, Validate: oneOf("small", "medium", "large"),- Help: "How much memory Fluxer's services may use"},+ Help: "A fixed amount of memory for Fluxer's services, instead of sizing them to this machine on every start"},{Name: EngineHost, Type: "endpoint", Validate: validEndpoint,Help: "The Docker or Podman engine that runs Fluxer"},}diff --git a/fluxer_rexctl/internal/stack/sizing.go b/fluxer_rexctl/internal/stack/sizing.gonew file mode 100644index 000000000..2329b6aa2--- /dev/null+++ b/fluxer_rexctl/internal/stack/sizing.go@@ -0,0 +1,158 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++package stack++import (+ "maps"+ "slices"+ "strconv"+ "strings"+)++type Hardware struct {+ MemoryBytes uint64+ CPUs int+}++const (+ mib = 1 << 20+ smallHostBytes = 7 << 30+ defaultWriterMiB = 7168+)++func Sizing(hw Hardware, memory bool, fixed map[string]string) map[string]string {+ out := map[string]string{}+ if hw.CPUs <= 0 {+ return out+ }+ get := func(k string) string {+ if v := strings.TrimSpace(fixed[k]); v != "" {+ return v+ }+ return out[k]+ }+ num := func(k string, fallback int) int {+ if n, err := strconv.Atoi(get(k)); err == nil && n > 0 {+ return n+ }+ return fallback+ }+ size := func(k string, fallback int) int {+ if n := parseMiB(get(k)); n > 0 {+ return n+ }+ return fallback+ }++ c := hw.CPUs+ small := min(c, clamp((c+3)/4, 2, 4))+ setInt(out, "FLUXER_TOKIO_WORKER_THREADS", small)+ setInt(out, "FLUXER_MEDIA_PROXY_TOKIO_WORKER_THREADS", min(c, clamp((c+1)/2, 2, 8)))+ setInt(out, "FLUXER_MALLOC_ARENA_MAX", max(2, small))+ setInt(out, "FLUXER_API_POSTGRES_MAX_CONNECTIONS", clamp(2*c, 4, 25))+ setInt(out, "FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS", clamp(c, 2, 25))+ setInt(out, "FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS", clamp(c, 2, 20))+ setInt(out, "FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS", clamp(c, 2, 20))+ clients := num("FLUXER_API_POSTGRES_MAX_CONNECTIONS", 25) + num("FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS", 25) ++ num("FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS", 20) + num("FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS", 20)+ setInt(out, "FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS", clients+10)+ setInt(out, "FLUXER_POSTGRES_AUTOVACUUM_MAX_WORKERS", clamp(c, 1, 3))+ gather := 0+ if c > 2 {+ gather = min(2, c/4)+ }+ setInt(out, "FLUXER_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER", gather)+ setInt(out, "FLUXER_POSTGRES_MAX_PARALLEL_WORKERS", min(8, c))+ setInt(out, "FLUXER_MEILISEARCH_MAX_INDEXING_THREADS", clamp(c/4, 1, 4))++ if !memory || hw.MemoryBytes == 0 {+ return out+ }+ total := int(hw.MemoryBytes / mib)+ m := max(1024, (total-max(256, total/10))&^255)+ share := func(k string, percent, lo, hi int) int {+ v := clamp(m*percent/100, lo, hi)+ out[k] = strconv.Itoa(v) + "mb"+ return size(k, v)+ }+ reserve := func(k string, limit int) {+ out[k] = strconv.Itoa(limit*2/5) + "mb"+ }++ reserve("FLUXER_API_MEMORY_RESERVATION", share("FLUXER_API_MEMORY_LIMIT", 20, 768, 2560))+ reserve("FLUXER_WORKER_MEMORY_RESERVATION", share("FLUXER_WORKER_MEMORY_LIMIT", 17, 640, 2560))+ reserve("FLUXER_GATEWAY_MEMORY_RESERVATION", share("FLUXER_GATEWAY_MEMORY_LIMIT", 12, 384, 4096))+ share("FLUXER_CADDY_MEMORY_LIMIT", 4, 256, 1024)+ if hw.MemoryBytes < smallHostBytes {+ out["FLUXER_EDGE_PROTOCOLS"] = "h1 h2"+ }++ media := share("FLUXER_MEDIA_PROXY_MEMORY_LIMIT", 8, 320, 1024)+ setBytes(out, "FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES", media/8)+ setBytes(out, "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES", media/4)+ setBytes(out, "FLUXER_GIFS_SHARD_CACHE_MAX_BYTES", min(clamp(m/64, 64, 128), size("FLUXER_GIFS_SHARD_MEMORY_LIMIT", 256)/2))++ out["FLUXER_LIVEKIT_GOMEMLIMIT"] = strconv.Itoa(share("FLUXER_LIVEKIT_MEMORY_LIMIT", 6, 320, 2048)*85/100) + "MiB"+ out["FLUXER_NATS_GOMEMLIMIT"] = strconv.Itoa(share("FLUXER_NATS_MEMORY_LIMIT", 3, 192, 1024)*4/5) + "MiB"+ out["FLUXER_SEAWEEDFS_GOMEMLIMIT"] = strconv.Itoa(share("FLUXER_SEAWEEDFS_MEMORY_LIMIT", 8, 448, 2048)*85/100) + "MiB"+ out["FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY"] = strconv.Itoa(share("FLUXER_MEILISEARCH_MEMORY_LIMIT", 15, 512, 4096)/3) + "mb"++ out["FLUXER_VALKEY_MEMORY_LIMIT"] = strconv.Itoa(clamp(m*45/1000+32, 224, 1568)) + "mb"+ out["FLUXER_VALKEY_MAXMEMORY"] = strconv.Itoa((size("FLUXER_VALKEY_MEMORY_LIMIT", 256)-32)*2/3) + "mb"++ pg := share("FLUXER_POSTGRES_MEMORY_LIMIT", 22, 448, 16384)+ reserve("FLUXER_POSTGRES_MEMORY_RESERVATION", pg)+ out["FLUXER_POSTGRES_SHARED_BUFFERS"] = strconv.Itoa(clamp(pg/4, 64, 4096)&^15) + "MB"+ buffers := size("FLUXER_POSTGRES_SHARED_BUFFERS", 512)+ out["FLUXER_POSTGRES_EFFECTIVE_CACHE_SIZE"] = strconv.Itoa(max(128, pg-buffers-64*c)) + "MB"+ work := clamp((pg-buffers)/(3*num("FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS", 150)), 2, 32)+ out["FLUXER_POSTGRES_WORK_MEM"] = strconv.Itoa(work) + "MB"+ out["FLUXER_POSTGRES_MAINTENANCE_WORK_MEM"] = strconv.Itoa(clamp(pg/16, 32, 1024)) + "MB"+ out["FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM"] = strconv.Itoa(clamp(pg/32, 16, 512)) + "MB"+ maintenance := size("FLUXER_POSTGRES_MAINTENANCE_WORK_MEM", 256)+ out["FLUXER_POSTGRES_SHM_SIZE"] = strconv.Itoa(64+maintenance+2*num("FLUXER_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER", 0)*work) + "mb"+ if m < defaultWriterMiB {+ out["FLUXER_POSTGRES_BGWRITER_DELAY"] = "200ms"+ out["FLUXER_POSTGRES_BGWRITER_LRU_MAXPAGES"] = "100"+ }+ return out+}++func SizingKeys() []string {+ return slices.Sorted(maps.Keys(Sizing(Hardware{MemoryBytes: 4 << 30, CPUs: 2}, true, nil)))+}++func clamp(v, lo, hi int) int { return max(lo, min(hi, v)) }++func setInt(out map[string]string, k string, v int) { out[k] = strconv.Itoa(v) }++func setBytes(out map[string]string, k string, mebibytes int) {+ out[k] = strconv.FormatInt(int64(mebibytes)*mib, 10)+}++func parseMiB(s string) int {+ s = strings.ToLower(strings.TrimSpace(s))+ i := strings.IndexFunc(s, func(r rune) bool { return r < '0' || r > '9' })+ if i == 0 {+ return 0+ }+ digits, unit := s, ""+ if i > 0 {+ digits, unit = s[:i], strings.TrimSpace(s[i:])+ }+ n, err := strconv.ParseInt(digits, 10, 64)+ if err != nil {+ return 0+ }+ switch strings.TrimSuffix(strings.TrimSuffix(unit, "b"), "i") {+ case "":+ return int(n / mib)+ case "k":+ return int(n / 1024)+ case "m":+ return int(n)+ case "g":+ return int(n * 1024)+ }+ return 0+}