SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
admin: configure maintenance tasks via admin.toml (#9926)
Chris Lu37962e244530
diff --git a/weed/admin/dash/config_persistence.go b/weed/admin/dash/config_persistence.goindex ef83638357..63d6bcdccb 100644--- a/weed/admin/dash/config_persistence.go+++ b/weed/admin/dash/config_persistence.go@@ -706,7 +706,7 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy {TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{VacuumConfig: &worker_pb.VacuumTaskConfig{GarbageThreshold: float64(vacuumConfig.GarbageThreshold),- MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours+ MinVolumeAgeHours: int32((vacuumConfig.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0},},}diff --git a/weed/admin/dash/config_toml.go b/weed/admin/dash/config_toml.gonew file mode 100644index 0000000000..81e31da723--- /dev/null+++ b/weed/admin/dash/config_toml.go@@ -0,0 +1,127 @@+package dash++import (+ "fmt"+ "strings"++ "github.com/seaweedfs/seaweedfs/weed/glog"+ "github.com/seaweedfs/seaweedfs/weed/util"+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"+)++// TomlConfig is the subset of viper used to read admin.toml values.+type TomlConfig interface {+ IsSet(key string) bool+ GetBool(key string) bool+ GetInt(key string) int+ GetFloat64(key string) float64+ GetString(key string) string+ GetStringSlice(key string) []string+}++// ApplyMaintenanceConfigFromToml writes maintenance task settings declared in+// admin.toml through to the persisted task configs, so they survive data+// directory loss and override admin UI edits on restart. Absent keys keep+// their persisted values.+func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error {+ vacuumConf := vacuum.LoadConfigFromPersistence(cp)+ vacuumChanged := applyBaseConfigFromToml(v, "maintenance.vacuum.", &vacuumConf.BaseConfig)+ if k := "maintenance.vacuum.garbage_threshold"; v.IsSet(k) {+ vacuumConf.GarbageThreshold = v.GetFloat64(k)+ vacuumChanged = true+ }+ if k := "maintenance.vacuum.min_volume_age_seconds"; v.IsSet(k) {+ vacuumConf.MinVolumeAgeSeconds = v.GetInt(k)+ vacuumChanged = true+ }++ balanceConf := balance.LoadConfigFromPersistence(cp)+ balanceChanged := applyBaseConfigFromToml(v, "maintenance.balance.", &balanceConf.BaseConfig)+ if k := "maintenance.balance.imbalance_threshold"; v.IsSet(k) {+ balanceConf.ImbalanceThreshold = v.GetFloat64(k)+ balanceChanged = true+ }+ if k := "maintenance.balance.min_server_count"; v.IsSet(k) {+ balanceConf.MinServerCount = v.GetInt(k)+ balanceChanged = true+ }++ ecConf := erasure_coding.LoadConfigFromPersistence(cp)+ ecChanged := applyBaseConfigFromToml(v, "maintenance.erasure_coding.", &ecConf.BaseConfig)+ if k := "maintenance.erasure_coding.fullness_ratio"; v.IsSet(k) {+ ecConf.FullnessRatio = v.GetFloat64(k)+ ecChanged = true+ }+ if k := "maintenance.erasure_coding.quiet_for_seconds"; v.IsSet(k) {+ ecConf.QuietForSeconds = v.GetInt(k)+ ecChanged = true+ }+ if k := "maintenance.erasure_coding.min_size_mb"; v.IsSet(k) {+ ecConf.MinSizeMB = v.GetInt(k)+ ecChanged = true+ }+ if k := "maintenance.erasure_coding.collection_filter"; v.IsSet(k) {+ ecConf.CollectionFilter = v.GetString(k)+ ecChanged = true+ }+ if k := "maintenance.erasure_coding.preferred_tags"; v.IsSet(k) {+ // viper does not split comma-separated values from env vars+ var tags []string+ for _, tag := range v.GetStringSlice(k) {+ tags = append(tags, strings.Split(tag, ",")...)+ }+ ecConf.PreferredTags = util.NormalizeTagList(tags)+ ecChanged = true+ }+ if k := "maintenance.erasure_coding.replica_placement"; v.IsSet(k) {+ ecConf.ReplicaPlacement = v.GetString(k)+ ecChanged = true+ }++ if !vacuumChanged && !balanceChanged && !ecChanged {+ return nil+ }+ if !cp.IsConfigured() {+ return fmt.Errorf("admin.toml maintenance settings require -dataDir to persist")+ }++ if vacuumChanged {+ if err := cp.SaveVacuumTaskPolicy(vacuumConf.ToTaskPolicy()); err != nil {+ return fmt.Errorf("save vacuum task config: %w", err)+ }+ glog.V(0).Infof("Applied [maintenance.vacuum] settings from admin.toml")+ }+ if balanceChanged {+ if err := cp.SaveBalanceTaskPolicy(balanceConf.ToTaskPolicy()); err != nil {+ return fmt.Errorf("save balance task config: %w", err)+ }+ glog.V(0).Infof("Applied [maintenance.balance] settings from admin.toml")+ }+ if ecChanged {+ if err := cp.SaveErasureCodingTaskPolicy(ecConf.ToTaskPolicy()); err != nil {+ return fmt.Errorf("save erasure coding task config: %w", err)+ }+ glog.V(0).Infof("Applied [maintenance.erasure_coding] settings from admin.toml")+ }+ return nil+}++func applyBaseConfigFromToml(v TomlConfig, prefix string, c *base.BaseConfig) bool {+ changed := false+ if k := prefix + "enabled"; v.IsSet(k) {+ c.Enabled = v.GetBool(k)+ changed = true+ }+ if k := prefix + "scan_interval_seconds"; v.IsSet(k) {+ c.ScanIntervalSeconds = v.GetInt(k)+ changed = true+ }+ if k := prefix + "max_concurrent"; v.IsSet(k) {+ c.MaxConcurrent = v.GetInt(k)+ changed = true+ }+ return changed+}diff --git a/weed/admin/dash/config_toml_test.go b/weed/admin/dash/config_toml_test.gonew file mode 100644index 0000000000..4e55a81194--- /dev/null+++ b/weed/admin/dash/config_toml_test.go@@ -0,0 +1,109 @@+package dash++import (+ "os"+ "path/filepath"+ "reflect"+ "strings"+ "testing"++ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"+ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"+ "github.com/spf13/viper"+)++func tomlConfig(t *testing.T, content string) *viper.Viper {+ t.Helper()+ v := viper.New()+ v.SetConfigType("toml")+ if err := v.ReadConfig(strings.NewReader(content)); err != nil {+ t.Fatalf("read toml: %v", err)+ }+ return v+}++func TestApplyMaintenanceConfigFromToml(t *testing.T) {+ dir := t.TempDir()+ cp := NewConfigPersistence(dir)++ // simulate an earlier UI edit+ saved := vacuum.NewDefaultConfig()+ saved.MaxConcurrent = 5+ saved.GarbageThreshold = 0.5+ if err := cp.SaveVacuumTaskPolicy(saved.ToTaskPolicy()); err != nil {+ t.Fatalf("save vacuum policy: %v", err)+ }++ balanceBefore := balance.LoadConfigFromPersistence(cp)++ v := tomlConfig(t, `+[maintenance.vacuum]+garbage_threshold = 0.03+min_volume_age_seconds = 1800++[maintenance.erasure_coding]+enabled = false+fullness_ratio = 0.8+preferred_tags = "Fast, ssd"+`)+ if err := cp.ApplyMaintenanceConfigFromToml(v); err != nil {+ t.Fatalf("apply: %v", err)+ }++ vacuumConf := vacuum.LoadConfigFromPersistence(cp)+ if vacuumConf.GarbageThreshold != 0.03 {+ t.Errorf("garbage threshold = %v, want 0.03", vacuumConf.GarbageThreshold)+ }+ // sub-hour values round up to a whole hour instead of truncating to 0+ if vacuumConf.MinVolumeAgeSeconds != 3600 {+ t.Errorf("min volume age = %v, want 3600", vacuumConf.MinVolumeAgeSeconds)+ }+ if vacuumConf.MaxConcurrent != 5 {+ t.Errorf("max concurrent = %v, want UI-saved 5 preserved", vacuumConf.MaxConcurrent)+ }++ ecConf := erasure_coding.LoadConfigFromPersistence(cp)+ if ecConf.Enabled {+ t.Errorf("ec enabled = true, want false")+ }+ if ecConf.FullnessRatio != 0.8 {+ t.Errorf("fullness ratio = %v, want 0.8", ecConf.FullnessRatio)+ }+ if !reflect.DeepEqual(ecConf.PreferredTags, []string{"fast", "ssd"}) {+ t.Errorf("preferred tags = %v, want [fast ssd]", ecConf.PreferredTags)+ }+ if ecConf.QuietForSeconds != 3600 {+ t.Errorf("quiet for = %v, want default 3600 preserved", ecConf.QuietForSeconds)+ }++ // balance section absent: nothing written+ if _, err := os.Stat(filepath.Join(dir, ConfigSubdir, BalanceTaskConfigFile)); !os.IsNotExist(err) {+ t.Errorf("balance config written without [maintenance.balance] section")+ }+ if got := balance.LoadConfigFromPersistence(cp); !reflect.DeepEqual(got, balanceBefore) {+ t.Errorf("balance config changed without [maintenance.balance] section")+ }+}++func TestApplyMaintenanceConfigFromTomlNoKeys(t *testing.T) {+ dir := t.TempDir()+ cp := NewConfigPersistence(dir)+ if err := cp.ApplyMaintenanceConfigFromToml(viper.New()); err != nil {+ t.Fatalf("apply: %v", err)+ }+ if _, err := os.Stat(filepath.Join(dir, ConfigSubdir)); !os.IsNotExist(err) {+ t.Errorf("conf dir created without any maintenance keys set")+ }+}++func TestApplyMaintenanceConfigFromTomlRequiresDataDir(t *testing.T) {+ cp := NewConfigPersistence("")+ v := tomlConfig(t, `+[maintenance.vacuum]+garbage_threshold = 0.03+`)+ if err := cp.ApplyMaintenanceConfigFromToml(v); err == nil {+ t.Errorf("expected error when maintenance keys are set without a data dir")+ }+}diff --git a/weed/admin/maintenance/maintenance_manager.go b/weed/admin/maintenance/maintenance_manager.goindex 9aaf2ebfa3..509cdc8991 100644--- a/weed/admin/maintenance/maintenance_manager.go+++ b/weed/admin/maintenance/maintenance_manager.go@@ -33,7 +33,7 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy {TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{VacuumConfig: &worker_pb.VacuumTaskConfig{GarbageThreshold: float64(vacuumConfig.GarbageThreshold),- MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours+ MinVolumeAgeHours: int32((vacuumConfig.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0},},}diff --git a/weed/command/admin.go b/weed/command/admin.goindex c3a699082a..daa1b5cdcc 100644--- a/weed/command/admin.go+++ b/weed/command/admin.go@@ -171,8 +171,17 @@ var cmdAdmin = &Command{- Metrics are disabled when -metricsPort is 0 (the default)- Example: weed admin -metricsPort=9327 -master="localhost:9333"+ Maintenance Configuration:+ - An optional admin.toml declares maintenance task settings+ ([maintenance.vacuum], [maintenance.balance], [maintenance.erasure_coding])+ - Settings in admin.toml are applied at every startup, overriding values+ saved from the admin UI, so they can be managed declaratively+ - Requires -dataDir; values can also be set via WEED_* environment+ variables, e.g. WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3+ - Generate example admin.toml: weed scaffold -config=admin+Configuration File:- - The security.toml file is read from ".", "$HOME/.seaweedfs/",+ - The security.toml and admin.toml files are read from ".", "$HOME/.seaweedfs/","/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order- Generate example security.toml: weed scaffold -config=security@@ -191,6 +200,9 @@ func runAdmin(cmd *Command, args []string) bool {// Load security configurationutil.LoadSecurityConfiguration()+ // Optional admin.toml with maintenance task settings+ util.LoadConfiguration("admin", false)+// Apply security.toml / env var fallbacks for credential flags.// CLI flags take precedence over security.toml / WEED_* env vars.applyViperFallback(cmd, a.adminUser, "adminUser", "admin.user")@@ -337,6 +349,12 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,glog.Infof("Data directory created/verified: %s", dataDir)}+ // Write maintenance task settings from admin.toml into the persisted+ // task configs before the server loads them+ if err := dash.NewConfigPersistence(dataDir).ApplyMaintenanceConfigFromToml(util.GetViper()); err != nil {+ return fmt.Errorf("apply admin.toml: %w", err)+ }+// Detect TLS configuration to set Secure cookie flagcookieSecure := viper.GetString("https.admin.key") != ""diff --git a/weed/command/scaffold.go b/weed/command/scaffold.goindex 3530bab844..ea087ed4ed 100644--- a/weed/command/scaffold.go+++ b/weed/command/scaffold.go@@ -14,7 +14,7 @@ func init() {}var cmdScaffold = &Command{- UsageLine: "scaffold -config=[filer|notification|replication|security|master|volume|shell|credential]",+ UsageLine: "scaffold -config=[filer|notification|replication|security|master|volume|shell|credential|admin]",Short: "generate basic configuration files",Long: `Generate configuration files with all possible configurations for you to customize.@@ -31,7 +31,7 @@ var cmdScaffold = &Command{var (outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")- config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master|volume|shell|credential] the configuration file to generate")+ config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master|volume|shell|credential|admin] the configuration file to generate"))func runScaffold(cmd *Command, args []string) bool {@@ -54,6 +54,8 @@ func runScaffold(cmd *Command, args []string) bool {content = scaffold.Shellcase "credential":content = scaffold.Credential+ case "admin":+ content = scaffold.Admin}if content == "" {println("need a valid -config option")diff --git a/weed/command/scaffold/admin.toml b/weed/command/scaffold/admin.tomlnew file mode 100644index 0000000000..679053e96d--- /dev/null+++ b/weed/command/scaffold/admin.toml@@ -0,0 +1,47 @@+# Put this file in one of the locations, with descending priority+# ./admin.toml+# $HOME/.seaweedfs/admin.toml+# /etc/seaweedfs/admin.toml+# this file is read by "weed admin"++# Maintenance task settings declared here are applied every time the admin+# server starts, overriding values saved from the admin UI under+# <dataDir>/conf. Commented-out keys keep their saved or default values.+# Requires -dataDir.+# Each value can also be set via environment variable, e.g.+# export WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3++[maintenance.vacuum]+# enabled = true+# vacuum volumes with more deleted content than this ratio+# garbage_threshold = 0.3+# how often to scan for volumes needing vacuum+# scan_interval_seconds = 7200+# max_concurrent = 2+# only vacuum volumes older than this (rounded up to whole hours)+# min_volume_age_seconds = 86400++[maintenance.balance]+# enabled = true+# trigger balancing when volume distribution exceeds this ratio+# imbalance_threshold = 0.2+# scan_interval_seconds = 1800+# max_concurrent = 1+# minimum number of volume servers before balancing+# min_server_count = 2++[maintenance.erasure_coding]+# enabled = true+# only erasure code volumes this full+# fullness_ratio = 0.95+# only erasure code volumes not modified for this long+# quiet_for_seconds = 3600+# scan_interval_seconds = 3600+# max_concurrent = 1+# min_size_mb = 30+# only process volumes from this collection ("" = all collections)+# collection_filter = ""+# disk tags preferred for shard placement+# preferred_tags = ["fast", "ssd"]+# EC shard placement constraint, e.g. "020"; empty uses the master default replication+# replica_placement = ""diff --git a/weed/command/scaffold/example.go b/weed/command/scaffold/example.goindex 55259fae83..de645ac8c3 100644--- a/weed/command/scaffold/example.go+++ b/weed/command/scaffold/example.go@@ -35,3 +35,6 @@ var Shell string//go:embed credential.tomlvar Credential string++//go:embed admin.toml+var Admin stringdiff --git a/weed/worker/tasks/vacuum/config.go b/weed/worker/tasks/vacuum/config.goindex a2a45d57d7..5d4469707b 100644--- a/weed/worker/tasks/vacuum/config.go+++ b/weed/worker/tasks/vacuum/config.go@@ -39,7 +39,7 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{VacuumConfig: &worker_pb.VacuumTaskConfig{GarbageThreshold: float64(c.GarbageThreshold),- MinVolumeAgeHours: int32(c.MinVolumeAgeSeconds / 3600), // Convert seconds to hours+ MinVolumeAgeHours: int32((c.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0},},}