Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

admin: configure maintenance tasks via admin.toml (#9926)

Chris Lu37962e244530
diff --git a/weed/admin/dash/config_persistence.go b/weed/admin/dash/config_persistence.go
index ef83638357..63d6bcdccb 100644
--- a/weed/admin/dash/config_persistence.go
+++ b/weed/admin/dash/config_persistence.go
@@ -706,7 +706,7 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy {
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
VacuumConfig: &worker_pb.VacuumTaskConfig{
GarbageThreshold: float64(vacuumConfig.GarbageThreshold),
- MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
+ MinVolumeAgeHours: int32((vacuumConfig.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0
},
},
}
diff --git a/weed/admin/dash/config_toml.go b/weed/admin/dash/config_toml.go
new file mode 100644
index 0000000000..81e31da723
--- /dev/null
+++ b/weed/admin/dash/config_toml.go
@@ -0,0 +1,127 @@
+package dash
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/seaweedfs/seaweedfs/weed/glog"
+ "github.com/seaweedfs/seaweedfs/weed/util"
+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+)
+
+// TomlConfig is the subset of viper used to read admin.toml values.
+type TomlConfig interface {
+ IsSet(key string) bool
+ GetBool(key string) bool
+ GetInt(key string) int
+ GetFloat64(key string) float64
+ GetString(key string) string
+ GetStringSlice(key string) []string
+}
+
+// ApplyMaintenanceConfigFromToml writes maintenance task settings declared in
+// admin.toml through to the persisted task configs, so they survive data
+// directory loss and override admin UI edits on restart. Absent keys keep
+// their persisted values.
+func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error {
+ vacuumConf := vacuum.LoadConfigFromPersistence(cp)
+ vacuumChanged := applyBaseConfigFromToml(v, "maintenance.vacuum.", &vacuumConf.BaseConfig)
+ if k := "maintenance.vacuum.garbage_threshold"; v.IsSet(k) {
+ vacuumConf.GarbageThreshold = v.GetFloat64(k)
+ vacuumChanged = true
+ }
+ if k := "maintenance.vacuum.min_volume_age_seconds"; v.IsSet(k) {
+ vacuumConf.MinVolumeAgeSeconds = v.GetInt(k)
+ vacuumChanged = true
+ }
+
+ balanceConf := balance.LoadConfigFromPersistence(cp)
+ balanceChanged := applyBaseConfigFromToml(v, "maintenance.balance.", &balanceConf.BaseConfig)
+ if k := "maintenance.balance.imbalance_threshold"; v.IsSet(k) {
+ balanceConf.ImbalanceThreshold = v.GetFloat64(k)
+ balanceChanged = true
+ }
+ if k := "maintenance.balance.min_server_count"; v.IsSet(k) {
+ balanceConf.MinServerCount = v.GetInt(k)
+ balanceChanged = true
+ }
+
+ ecConf := erasure_coding.LoadConfigFromPersistence(cp)
+ ecChanged := applyBaseConfigFromToml(v, "maintenance.erasure_coding.", &ecConf.BaseConfig)
+ if k := "maintenance.erasure_coding.fullness_ratio"; v.IsSet(k) {
+ ecConf.FullnessRatio = v.GetFloat64(k)
+ ecChanged = true
+ }
+ if k := "maintenance.erasure_coding.quiet_for_seconds"; v.IsSet(k) {
+ ecConf.QuietForSeconds = v.GetInt(k)
+ ecChanged = true
+ }
+ if k := "maintenance.erasure_coding.min_size_mb"; v.IsSet(k) {
+ ecConf.MinSizeMB = v.GetInt(k)
+ ecChanged = true
+ }
+ if k := "maintenance.erasure_coding.collection_filter"; v.IsSet(k) {
+ ecConf.CollectionFilter = v.GetString(k)
+ ecChanged = true
+ }
+ if k := "maintenance.erasure_coding.preferred_tags"; v.IsSet(k) {
+ // viper does not split comma-separated values from env vars
+ var tags []string
+ for _, tag := range v.GetStringSlice(k) {
+ tags = append(tags, strings.Split(tag, ",")...)
+ }
+ ecConf.PreferredTags = util.NormalizeTagList(tags)
+ ecChanged = true
+ }
+ if k := "maintenance.erasure_coding.replica_placement"; v.IsSet(k) {
+ ecConf.ReplicaPlacement = v.GetString(k)
+ ecChanged = true
+ }
+
+ if !vacuumChanged && !balanceChanged && !ecChanged {
+ return nil
+ }
+ if !cp.IsConfigured() {
+ return fmt.Errorf("admin.toml maintenance settings require -dataDir to persist")
+ }
+
+ if vacuumChanged {
+ if err := cp.SaveVacuumTaskPolicy(vacuumConf.ToTaskPolicy()); err != nil {
+ return fmt.Errorf("save vacuum task config: %w", err)
+ }
+ glog.V(0).Infof("Applied [maintenance.vacuum] settings from admin.toml")
+ }
+ if balanceChanged {
+ if err := cp.SaveBalanceTaskPolicy(balanceConf.ToTaskPolicy()); err != nil {
+ return fmt.Errorf("save balance task config: %w", err)
+ }
+ glog.V(0).Infof("Applied [maintenance.balance] settings from admin.toml")
+ }
+ if ecChanged {
+ if err := cp.SaveErasureCodingTaskPolicy(ecConf.ToTaskPolicy()); err != nil {
+ return fmt.Errorf("save erasure coding task config: %w", err)
+ }
+ glog.V(0).Infof("Applied [maintenance.erasure_coding] settings from admin.toml")
+ }
+ return nil
+}
+
+func applyBaseConfigFromToml(v TomlConfig, prefix string, c *base.BaseConfig) bool {
+ changed := false
+ if k := prefix + "enabled"; v.IsSet(k) {
+ c.Enabled = v.GetBool(k)
+ changed = true
+ }
+ if k := prefix + "scan_interval_seconds"; v.IsSet(k) {
+ c.ScanIntervalSeconds = v.GetInt(k)
+ changed = true
+ }
+ if k := prefix + "max_concurrent"; v.IsSet(k) {
+ c.MaxConcurrent = v.GetInt(k)
+ changed = true
+ }
+ return changed
+}
diff --git a/weed/admin/dash/config_toml_test.go b/weed/admin/dash/config_toml_test.go
new file mode 100644
index 0000000000..4e55a81194
--- /dev/null
+++ b/weed/admin/dash/config_toml_test.go
@@ -0,0 +1,109 @@
+package dash
+
+import (
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+ "github.com/spf13/viper"
+)
+
+func tomlConfig(t *testing.T, content string) *viper.Viper {
+ t.Helper()
+ v := viper.New()
+ v.SetConfigType("toml")
+ if err := v.ReadConfig(strings.NewReader(content)); err != nil {
+ t.Fatalf("read toml: %v", err)
+ }
+ return v
+}
+
+func TestApplyMaintenanceConfigFromToml(t *testing.T) {
+ dir := t.TempDir()
+ cp := NewConfigPersistence(dir)
+
+ // simulate an earlier UI edit
+ saved := vacuum.NewDefaultConfig()
+ saved.MaxConcurrent = 5
+ saved.GarbageThreshold = 0.5
+ if err := cp.SaveVacuumTaskPolicy(saved.ToTaskPolicy()); err != nil {
+ t.Fatalf("save vacuum policy: %v", err)
+ }
+
+ balanceBefore := balance.LoadConfigFromPersistence(cp)
+
+ v := tomlConfig(t, `
+[maintenance.vacuum]
+garbage_threshold = 0.03
+min_volume_age_seconds = 1800
+
+[maintenance.erasure_coding]
+enabled = false
+fullness_ratio = 0.8
+preferred_tags = "Fast, ssd"
+`)
+ if err := cp.ApplyMaintenanceConfigFromToml(v); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+
+ vacuumConf := vacuum.LoadConfigFromPersistence(cp)
+ if vacuumConf.GarbageThreshold != 0.03 {
+ t.Errorf("garbage threshold = %v, want 0.03", vacuumConf.GarbageThreshold)
+ }
+ // sub-hour values round up to a whole hour instead of truncating to 0
+ if vacuumConf.MinVolumeAgeSeconds != 3600 {
+ t.Errorf("min volume age = %v, want 3600", vacuumConf.MinVolumeAgeSeconds)
+ }
+ if vacuumConf.MaxConcurrent != 5 {
+ t.Errorf("max concurrent = %v, want UI-saved 5 preserved", vacuumConf.MaxConcurrent)
+ }
+
+ ecConf := erasure_coding.LoadConfigFromPersistence(cp)
+ if ecConf.Enabled {
+ t.Errorf("ec enabled = true, want false")
+ }
+ if ecConf.FullnessRatio != 0.8 {
+ t.Errorf("fullness ratio = %v, want 0.8", ecConf.FullnessRatio)
+ }
+ if !reflect.DeepEqual(ecConf.PreferredTags, []string{"fast", "ssd"}) {
+ t.Errorf("preferred tags = %v, want [fast ssd]", ecConf.PreferredTags)
+ }
+ if ecConf.QuietForSeconds != 3600 {
+ t.Errorf("quiet for = %v, want default 3600 preserved", ecConf.QuietForSeconds)
+ }
+
+ // balance section absent: nothing written
+ if _, err := os.Stat(filepath.Join(dir, ConfigSubdir, BalanceTaskConfigFile)); !os.IsNotExist(err) {
+ t.Errorf("balance config written without [maintenance.balance] section")
+ }
+ if got := balance.LoadConfigFromPersistence(cp); !reflect.DeepEqual(got, balanceBefore) {
+ t.Errorf("balance config changed without [maintenance.balance] section")
+ }
+}
+
+func TestApplyMaintenanceConfigFromTomlNoKeys(t *testing.T) {
+ dir := t.TempDir()
+ cp := NewConfigPersistence(dir)
+ if err := cp.ApplyMaintenanceConfigFromToml(viper.New()); err != nil {
+ t.Fatalf("apply: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(dir, ConfigSubdir)); !os.IsNotExist(err) {
+ t.Errorf("conf dir created without any maintenance keys set")
+ }
+}
+
+func TestApplyMaintenanceConfigFromTomlRequiresDataDir(t *testing.T) {
+ cp := NewConfigPersistence("")
+ v := tomlConfig(t, `
+[maintenance.vacuum]
+garbage_threshold = 0.03
+`)
+ if err := cp.ApplyMaintenanceConfigFromToml(v); err == nil {
+ t.Errorf("expected error when maintenance keys are set without a data dir")
+ }
+}
diff --git a/weed/admin/maintenance/maintenance_manager.go b/weed/admin/maintenance/maintenance_manager.go
index 9aaf2ebfa3..509cdc8991 100644
--- a/weed/admin/maintenance/maintenance_manager.go
+++ b/weed/admin/maintenance/maintenance_manager.go
@@ -33,7 +33,7 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy {
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
VacuumConfig: &worker_pb.VacuumTaskConfig{
GarbageThreshold: float64(vacuumConfig.GarbageThreshold),
- MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
+ MinVolumeAgeHours: int32((vacuumConfig.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0
},
},
}
diff --git a/weed/command/admin.go b/weed/command/admin.go
index c3a699082a..daa1b5cdcc 100644
--- a/weed/command/admin.go
+++ b/weed/command/admin.go
@@ -171,8 +171,17 @@ var cmdAdmin = &Command{
- Metrics are disabled when -metricsPort is 0 (the default)
- Example: weed admin -metricsPort=9327 -master="localhost:9333"
+ Maintenance Configuration:
+ - An optional admin.toml declares maintenance task settings
+ ([maintenance.vacuum], [maintenance.balance], [maintenance.erasure_coding])
+ - Settings in admin.toml are applied at every startup, overriding values
+ saved from the admin UI, so they can be managed declaratively
+ - Requires -dataDir; values can also be set via WEED_* environment
+ variables, e.g. WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3
+ - Generate example admin.toml: weed scaffold -config=admin
+
Configuration File:
- - The security.toml file is read from ".", "$HOME/.seaweedfs/",
+ - The security.toml and admin.toml files are read from ".", "$HOME/.seaweedfs/",
"/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order
- Generate example security.toml: weed scaffold -config=security
@@ -191,6 +200,9 @@ func runAdmin(cmd *Command, args []string) bool {
// Load security configuration
util.LoadSecurityConfiguration()
+ // Optional admin.toml with maintenance task settings
+ util.LoadConfiguration("admin", false)
+
// Apply security.toml / env var fallbacks for credential flags.
// CLI flags take precedence over security.toml / WEED_* env vars.
applyViperFallback(cmd, a.adminUser, "adminUser", "admin.user")
@@ -337,6 +349,12 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
glog.Infof("Data directory created/verified: %s", dataDir)
}
+ // Write maintenance task settings from admin.toml into the persisted
+ // task configs before the server loads them
+ if err := dash.NewConfigPersistence(dataDir).ApplyMaintenanceConfigFromToml(util.GetViper()); err != nil {
+ return fmt.Errorf("apply admin.toml: %w", err)
+ }
+
// Detect TLS configuration to set Secure cookie flag
cookieSecure := viper.GetString("https.admin.key") != ""
diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index 3530bab844..ea087ed4ed 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -14,7 +14,7 @@ func init() {
}
var cmdScaffold = &Command{
- UsageLine: "scaffold -config=[filer|notification|replication|security|master|volume|shell|credential]",
+ UsageLine: "scaffold -config=[filer|notification|replication|security|master|volume|shell|credential|admin]",
Short: "generate basic configuration files",
Long: `Generate configuration files with all possible configurations for you to customize.
@@ -31,7 +31,7 @@ var cmdScaffold = &Command{
var (
outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
- config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master|volume|shell|credential] the configuration file to generate")
+ config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master|volume|shell|credential|admin] the configuration file to generate")
)
func runScaffold(cmd *Command, args []string) bool {
@@ -54,6 +54,8 @@ func runScaffold(cmd *Command, args []string) bool {
content = scaffold.Shell
case "credential":
content = scaffold.Credential
+ case "admin":
+ content = scaffold.Admin
}
if content == "" {
println("need a valid -config option")
diff --git a/weed/command/scaffold/admin.toml b/weed/command/scaffold/admin.toml
new file mode 100644
index 0000000000..679053e96d
--- /dev/null
+++ b/weed/command/scaffold/admin.toml
@@ -0,0 +1,47 @@
+# Put this file in one of the locations, with descending priority
+# ./admin.toml
+# $HOME/.seaweedfs/admin.toml
+# /etc/seaweedfs/admin.toml
+# this file is read by "weed admin"
+
+# Maintenance task settings declared here are applied every time the admin
+# server starts, overriding values saved from the admin UI under
+# <dataDir>/conf. Commented-out keys keep their saved or default values.
+# Requires -dataDir.
+# Each value can also be set via environment variable, e.g.
+# export WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3
+
+[maintenance.vacuum]
+# enabled = true
+# vacuum volumes with more deleted content than this ratio
+# garbage_threshold = 0.3
+# how often to scan for volumes needing vacuum
+# scan_interval_seconds = 7200
+# max_concurrent = 2
+# only vacuum volumes older than this (rounded up to whole hours)
+# min_volume_age_seconds = 86400
+
+[maintenance.balance]
+# enabled = true
+# trigger balancing when volume distribution exceeds this ratio
+# imbalance_threshold = 0.2
+# scan_interval_seconds = 1800
+# max_concurrent = 1
+# minimum number of volume servers before balancing
+# min_server_count = 2
+
+[maintenance.erasure_coding]
+# enabled = true
+# only erasure code volumes this full
+# fullness_ratio = 0.95
+# only erasure code volumes not modified for this long
+# quiet_for_seconds = 3600
+# scan_interval_seconds = 3600
+# max_concurrent = 1
+# min_size_mb = 30
+# only process volumes from this collection ("" = all collections)
+# collection_filter = ""
+# disk tags preferred for shard placement
+# preferred_tags = ["fast", "ssd"]
+# EC shard placement constraint, e.g. "020"; empty uses the master default replication
+# replica_placement = ""
diff --git a/weed/command/scaffold/example.go b/weed/command/scaffold/example.go
index 55259fae83..de645ac8c3 100644
--- a/weed/command/scaffold/example.go
+++ b/weed/command/scaffold/example.go
@@ -35,3 +35,6 @@ var Shell string
//go:embed credential.toml
var Credential string
+
+//go:embed admin.toml
+var Admin string
diff --git a/weed/worker/tasks/vacuum/config.go b/weed/worker/tasks/vacuum/config.go
index a2a45d57d7..5d4469707b 100644
--- a/weed/worker/tasks/vacuum/config.go
+++ b/weed/worker/tasks/vacuum/config.go
@@ -39,7 +39,7 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
VacuumConfig: &worker_pb.VacuumTaskConfig{
GarbageThreshold: float64(c.GarbageThreshold),
- MinVolumeAgeHours: int32(c.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
+ MinVolumeAgeHours: int32((c.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0
},
},
}