Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

feat(rexctl): size services from memory and cpus on every start

Hampusdb4cbe501027
diff --git a/fluxer_docs/src/content/docs/operator/rexctl.mdx b/fluxer_docs/src/content/docs/operator/rexctl.mdx
index 4e9969a27..6ccc95878 100644
--- a/fluxer_docs/src/content/docs/operator/rexctl.mdx
+++ b/fluxer_docs/src/content/docs/operator/rexctl.mdx
@@ -290,7 +290,7 @@ rexctl install [flags]
- `--memory-profile string`
- Small, medium or large (picked from this machine's memory).
+ Pin small, medium or large instead of sizing Fluxer to this machine on every start.
- `--name string`
diff --git a/fluxer_docs/src/content/docs/operator/running.mdx b/fluxer_docs/src/content/docs/operator/running.mdx
index b27b8bf67..c3abe270e 100644
--- a/fluxer_docs/src/content/docs/operator/running.mdx
+++ b/fluxer_docs/src/content/docs/operator/running.mdx
@@ -107,6 +107,8 @@ This lists every setting with its value and a line of help, and says which setti
rexctl config set <key> <value>
```
+Each time rexctl starts Fluxer, it sizes memory limits, database connections and thread counts to the machine's memory and CPUs, so a resized machine needs no changes. The `memory.profile` setting pins a fixed size instead. An instance installed by an older rexctl keeps the size it picked at install. Run `rexctl config unset memory.profile` to let rexctl size it.
+
## When something is wrong
```sh
diff --git a/fluxer_rexctl/cmd/install.go b/fluxer_rexctl/cmd/install.go
index b2a3c2622..dfd3d672f 100644
--- a/fluxer_rexctl/cmd/install.go
+++ b/fluxer_rexctl/cmd/install.go
@@ -88,7 +88,7 @@ func newInstallCmd(app *App) *cobra.Command {
fl.BoolVar(&f.BehindProxy, "behind-proxy", false, "serve plain HTTP for your own reverse proxy")
fl.StringVar(&f.Bind, "bind", "127.0.0.1:8080", "address your proxy connects to, with --behind-proxy")
fl.StringVar(&f.VoiceIP, "voice-ip", "", "public IP address for voice and video (found automatically)")
- fl.StringVar(&f.MemoryProfile, "memory-profile", "", "small, medium or large (picked from this machine's memory)")
+ fl.StringVar(&f.MemoryProfile, "memory-profile", "", "pin small, medium or large instead of sizing Fluxer to this machine on every start")
fl.BoolVar(&f.InstallDocker, "install-docker", true, "install Docker if it's missing")
fl.BoolVar(&f.OpenFirewall, "open-firewall", true, "open the ports Fluxer needs in ufw or firewalld")
fl.BoolVar(&f.SetupPodman, "setup-podman", true, "start Podman's API socket and start Fluxer with the machine")
@@ -856,8 +856,6 @@ func (in *installer) write(ctx context.Context, inst *instance.Instance, deleteO
}
if f.MemoryProfile != "" {
st.Settings[settings.MemoryProfile] = f.MemoryProfile
- } else if in.host != nil && in.host.MemoryBytes > 0 {
- st.Settings[settings.MemoryProfile] = string(stack.ProfileFor(in.host.MemoryBytes))
}
if _, err := secrets.Mint(st.Secrets, false); err != nil {
return ui.Wrap(err, "Couldn't create your instance's secrets.", "Run rexctl install again.")
diff --git a/fluxer_rexctl/internal/deploy/set.go b/fluxer_rexctl/internal/deploy/set.go
index 165e1d549..1177b3b74 100644
--- a/fluxer_rexctl/internal/deploy/set.go
+++ b/fluxer_rexctl/internal/deploy/set.go
@@ -37,7 +37,7 @@ func Promised(set *stack.Set, custom []byte, k settings.Key) []string {
case settings.EdgeMode, settings.IPv6, settings.ImagesRegistry, settings.StorageEndpoint:
return slices.Clone(stack.Services)
case settings.MemoryProfile:
- return settings.Services(set, custom, stack.ProfileKeys(set))
+ return settings.Services(set, custom, append(stack.ProfileKeys(set), stack.SizingKeys()...))
}
return settings.Services(set, custom, append(slices.Clone(k.Vars), derivedVars[k.Name]...))
}
diff --git a/fluxer_rexctl/internal/host/cgroup_linux.go b/fluxer_rexctl/internal/host/cgroup_linux.go
new file mode 100644
index 000000000..1f2fa9b2e
--- /dev/null
+++ b/fluxer_rexctl/internal/host/cgroup_linux.go
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package host
+
+import (
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+)
+
+const cgroupRoot = "/sys/fs/cgroup"
+
+func cgroupLimits() (memoryBytes uint64, cpus int) {
+ data, err := os.ReadFile("/proc/self/cgroup")
+ if err != nil {
+ return 0, 0
+ }
+ var own string
+ for line := range strings.Lines(string(data)) {
+ if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "0::"); ok {
+ own = rest
+ break
+ }
+ }
+ if own == "" {
+ return 0, 0
+ }
+ dir := filepath.Join(cgroupRoot, own)
+ for strings.HasPrefix(dir, cgroupRoot) {
+ if v, ok := cgroupMemoryMax(filepath.Join(dir, "memory.max")); ok && (memoryBytes == 0 || v < memoryBytes) {
+ memoryBytes = v
+ }
+ if n := cgroupCPUMax(filepath.Join(dir, "cpu.max")); n > 0 && (cpus == 0 || n < cpus) {
+ cpus = n
+ }
+ if dir == cgroupRoot {
+ break
+ }
+ dir = filepath.Dir(dir)
+ }
+ return memoryBytes, cpus
+}
+
+func cgroupMemoryMax(path string) (uint64, bool) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return 0, false
+ }
+ v, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
+ return v, err == nil && v > 0
+}
+
+func cgroupCPUMax(path string) int {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return 0
+ }
+ f := strings.Fields(string(data))
+ if len(f) != 2 {
+ return 0
+ }
+ quota, err1 := strconv.ParseUint(f[0], 10, 64)
+ period, err2 := strconv.ParseUint(f[1], 10, 64)
+ if err1 != nil || err2 != nil || period == 0 {
+ return 0
+ }
+ return int((quota + period - 1) / period)
+}
diff --git a/fluxer_rexctl/internal/host/cgroup_other.go b/fluxer_rexctl/internal/host/cgroup_other.go
new file mode 100644
index 000000000..cfc761c7a
--- /dev/null
+++ b/fluxer_rexctl/internal/host/cgroup_other.go
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+//go:build !linux
+
+package host
+
+func cgroupLimits() (memoryBytes uint64, cpus int) { return 0, 0 }
diff --git a/fluxer_rexctl/internal/host/hardware.go b/fluxer_rexctl/internal/host/hardware.go
new file mode 100644
index 000000000..3849c1c5e
--- /dev/null
+++ b/fluxer_rexctl/internal/host/hardware.go
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package host
+
+import "runtime"
+
+func Hardware() (memoryBytes uint64, cpus int) {
+ memoryBytes, _ = Memory()
+ cpus = runtime.NumCPU()
+ limit, quota := cgroupLimits()
+ if limit > 0 && (memoryBytes == 0 || limit < memoryBytes) {
+ memoryBytes = limit
+ }
+ if quota > 0 && quota < cpus {
+ cpus = quota
+ }
+ return memoryBytes, cpus
+}
diff --git a/fluxer_rexctl/internal/settings/effective.go b/fluxer_rexctl/internal/settings/effective.go
index 2074627e9..19d611601 100644
--- a/fluxer_rexctl/internal/settings/effective.go
+++ b/fluxer_rexctl/internal/settings/effective.go
@@ -8,7 +8,9 @@ import (
"slices"
"strconv"
"strings"
+ "sync"
+ "github.com/fluxerapp/fluxer/fluxer_rexctl/internal/host"
"github.com/fluxerapp/fluxer/fluxer_rexctl/internal/stack"
)
@@ -21,6 +23,12 @@ func Effective(set *stack.Set, chosen, secrets, adminImport map[string]string) (
}
}
+ if len(set.Profiles) > 0 {
+ for v, value := range stack.Sizing(Hardware(), chosen[MemoryProfile] == "", fixed) {
+ put(v, value)
+ }
+ }
+
if name := chosen[MemoryProfile]; name != "" {
values, ok := set.Profiles[name]
if !ok {
@@ -78,6 +86,11 @@ func Effective(set *stack.Set, chosen, secrets, adminImport map[string]string) (
return env, nil
}
+var Hardware = sync.OnceValue(func() stack.Hardware {
+ memory, cpus := host.Hardware()
+ return stack.Hardware{MemoryBytes: memory, CPUs: cpus}
+})
+
func namedVars(k Key, value string) map[string]string {
switch k.Name {
case VoicePublicIP:
diff --git a/fluxer_rexctl/internal/settings/keys.go b/fluxer_rexctl/internal/settings/keys.go
index 0ff9cea72..f08adab97 100644
--- a/fluxer_rexctl/internal/settings/keys.go
+++ b/fluxer_rexctl/internal/settings/keys.go
@@ -128,7 +128,7 @@ func buildNamed() []Key {
{Name: ImagesRegistry, Type: "registry", Validate: validRegistry,
Help: "A registry mirror for the database, cache, search, storage, voice and edge images"},
{Name: MemoryProfile, Type: "choice", Options: []string{"small", "medium", "large"}, Validate: oneOf("small", "medium", "large"),
- Help: "How much memory Fluxer's services may use"},
+ Help: "A fixed amount of memory for Fluxer's services, instead of sizing them to this machine on every start"},
{Name: EngineHost, Type: "endpoint", Validate: validEndpoint,
Help: "The Docker or Podman engine that runs Fluxer"},
}
diff --git a/fluxer_rexctl/internal/stack/sizing.go b/fluxer_rexctl/internal/stack/sizing.go
new file mode 100644
index 000000000..2329b6aa2
--- /dev/null
+++ b/fluxer_rexctl/internal/stack/sizing.go
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package stack
+
+import (
+ "maps"
+ "slices"
+ "strconv"
+ "strings"
+)
+
+type Hardware struct {
+ MemoryBytes uint64
+ CPUs int
+}
+
+const (
+ mib = 1 << 20
+ smallHostBytes = 7 << 30
+ defaultWriterMiB = 7168
+)
+
+func Sizing(hw Hardware, memory bool, fixed map[string]string) map[string]string {
+ out := map[string]string{}
+ if hw.CPUs <= 0 {
+ return out
+ }
+ get := func(k string) string {
+ if v := strings.TrimSpace(fixed[k]); v != "" {
+ return v
+ }
+ return out[k]
+ }
+ num := func(k string, fallback int) int {
+ if n, err := strconv.Atoi(get(k)); err == nil && n > 0 {
+ return n
+ }
+ return fallback
+ }
+ size := func(k string, fallback int) int {
+ if n := parseMiB(get(k)); n > 0 {
+ return n
+ }
+ return fallback
+ }
+
+ c := hw.CPUs
+ small := min(c, clamp((c+3)/4, 2, 4))
+ setInt(out, "FLUXER_TOKIO_WORKER_THREADS", small)
+ setInt(out, "FLUXER_MEDIA_PROXY_TOKIO_WORKER_THREADS", min(c, clamp((c+1)/2, 2, 8)))
+ setInt(out, "FLUXER_MALLOC_ARENA_MAX", max(2, small))
+ setInt(out, "FLUXER_API_POSTGRES_MAX_CONNECTIONS", clamp(2*c, 4, 25))
+ setInt(out, "FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS", clamp(c, 2, 25))
+ setInt(out, "FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS", clamp(c, 2, 20))
+ setInt(out, "FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS", clamp(c, 2, 20))
+ clients := num("FLUXER_API_POSTGRES_MAX_CONNECTIONS", 25) + num("FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS", 25) +
+ num("FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS", 20) + num("FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS", 20)
+ setInt(out, "FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS", clients+10)
+ setInt(out, "FLUXER_POSTGRES_AUTOVACUUM_MAX_WORKERS", clamp(c, 1, 3))
+ gather := 0
+ if c > 2 {
+ gather = min(2, c/4)
+ }
+ setInt(out, "FLUXER_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER", gather)
+ setInt(out, "FLUXER_POSTGRES_MAX_PARALLEL_WORKERS", min(8, c))
+ setInt(out, "FLUXER_MEILISEARCH_MAX_INDEXING_THREADS", clamp(c/4, 1, 4))
+
+ if !memory || hw.MemoryBytes == 0 {
+ return out
+ }
+ total := int(hw.MemoryBytes / mib)
+ m := max(1024, (total-max(256, total/10))&^255)
+ share := func(k string, percent, lo, hi int) int {
+ v := clamp(m*percent/100, lo, hi)
+ out[k] = strconv.Itoa(v) + "mb"
+ return size(k, v)
+ }
+ reserve := func(k string, limit int) {
+ out[k] = strconv.Itoa(limit*2/5) + "mb"
+ }
+
+ reserve("FLUXER_API_MEMORY_RESERVATION", share("FLUXER_API_MEMORY_LIMIT", 20, 768, 2560))
+ reserve("FLUXER_WORKER_MEMORY_RESERVATION", share("FLUXER_WORKER_MEMORY_LIMIT", 17, 640, 2560))
+ reserve("FLUXER_GATEWAY_MEMORY_RESERVATION", share("FLUXER_GATEWAY_MEMORY_LIMIT", 12, 384, 4096))
+ share("FLUXER_CADDY_MEMORY_LIMIT", 4, 256, 1024)
+ if hw.MemoryBytes < smallHostBytes {
+ out["FLUXER_EDGE_PROTOCOLS"] = "h1 h2"
+ }
+
+ media := share("FLUXER_MEDIA_PROXY_MEMORY_LIMIT", 8, 320, 1024)
+ setBytes(out, "FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES", media/8)
+ setBytes(out, "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES", media/4)
+ setBytes(out, "FLUXER_GIFS_SHARD_CACHE_MAX_BYTES", min(clamp(m/64, 64, 128), size("FLUXER_GIFS_SHARD_MEMORY_LIMIT", 256)/2))
+
+ out["FLUXER_LIVEKIT_GOMEMLIMIT"] = strconv.Itoa(share("FLUXER_LIVEKIT_MEMORY_LIMIT", 6, 320, 2048)*85/100) + "MiB"
+ out["FLUXER_NATS_GOMEMLIMIT"] = strconv.Itoa(share("FLUXER_NATS_MEMORY_LIMIT", 3, 192, 1024)*4/5) + "MiB"
+ out["FLUXER_SEAWEEDFS_GOMEMLIMIT"] = strconv.Itoa(share("FLUXER_SEAWEEDFS_MEMORY_LIMIT", 8, 448, 2048)*85/100) + "MiB"
+ out["FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY"] = strconv.Itoa(share("FLUXER_MEILISEARCH_MEMORY_LIMIT", 15, 512, 4096)/3) + "mb"
+
+ out["FLUXER_VALKEY_MEMORY_LIMIT"] = strconv.Itoa(clamp(m*45/1000+32, 224, 1568)) + "mb"
+ out["FLUXER_VALKEY_MAXMEMORY"] = strconv.Itoa((size("FLUXER_VALKEY_MEMORY_LIMIT", 256)-32)*2/3) + "mb"
+
+ pg := share("FLUXER_POSTGRES_MEMORY_LIMIT", 22, 448, 16384)
+ reserve("FLUXER_POSTGRES_MEMORY_RESERVATION", pg)
+ out["FLUXER_POSTGRES_SHARED_BUFFERS"] = strconv.Itoa(clamp(pg/4, 64, 4096)&^15) + "MB"
+ buffers := size("FLUXER_POSTGRES_SHARED_BUFFERS", 512)
+ out["FLUXER_POSTGRES_EFFECTIVE_CACHE_SIZE"] = strconv.Itoa(max(128, pg-buffers-64*c)) + "MB"
+ work := clamp((pg-buffers)/(3*num("FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS", 150)), 2, 32)
+ out["FLUXER_POSTGRES_WORK_MEM"] = strconv.Itoa(work) + "MB"
+ out["FLUXER_POSTGRES_MAINTENANCE_WORK_MEM"] = strconv.Itoa(clamp(pg/16, 32, 1024)) + "MB"
+ out["FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM"] = strconv.Itoa(clamp(pg/32, 16, 512)) + "MB"
+ maintenance := size("FLUXER_POSTGRES_MAINTENANCE_WORK_MEM", 256)
+ out["FLUXER_POSTGRES_SHM_SIZE"] = strconv.Itoa(64+maintenance+2*num("FLUXER_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER", 0)*work) + "mb"
+ if m < defaultWriterMiB {
+ out["FLUXER_POSTGRES_BGWRITER_DELAY"] = "200ms"
+ out["FLUXER_POSTGRES_BGWRITER_LRU_MAXPAGES"] = "100"
+ }
+ return out
+}
+
+func SizingKeys() []string {
+ return slices.Sorted(maps.Keys(Sizing(Hardware{MemoryBytes: 4 << 30, CPUs: 2}, true, nil)))
+}
+
+func clamp(v, lo, hi int) int { return max(lo, min(hi, v)) }
+
+func setInt(out map[string]string, k string, v int) { out[k] = strconv.Itoa(v) }
+
+func setBytes(out map[string]string, k string, mebibytes int) {
+ out[k] = strconv.FormatInt(int64(mebibytes)*mib, 10)
+}
+
+func parseMiB(s string) int {
+ s = strings.ToLower(strings.TrimSpace(s))
+ i := strings.IndexFunc(s, func(r rune) bool { return r < '0' || r > '9' })
+ if i == 0 {
+ return 0
+ }
+ digits, unit := s, ""
+ if i > 0 {
+ digits, unit = s[:i], strings.TrimSpace(s[i:])
+ }
+ n, err := strconv.ParseInt(digits, 10, 64)
+ if err != nil {
+ return 0
+ }
+ switch strings.TrimSuffix(strings.TrimSuffix(unit, "b"), "i") {
+ case "":
+ return int(n / mib)
+ case "k":
+ return int(n / 1024)
+ case "m":
+ return int(n)
+ case "g":
+ return int(n * 1024)
+ }
+ return 0
+}