Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fs: add Options registry and rework rc to use it

Nick Craig-Woode79273f9c984
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 203253601..835033ef9 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -29,7 +29,6 @@ import (
"github.com/rclone/rclone/fs/config/configflags"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/filter"
- "github.com/rclone/rclone/fs/filter/filterflags"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fspath"
fslog "github.com/rclone/rclone/fs/log"
@@ -383,6 +382,12 @@ func StartStats() func() {
// initConfig is run by cobra after initialising the flags
func initConfig() {
+ // Set the global options from the flags
+ err := fs.GlobalOptionsInit()
+ if err != nil {
+ log.Fatalf("Failed to initialise global options: %v", err)
+ }
+
ctx := context.Background()
ci := fs.GetConfig(ctx)
@@ -409,12 +414,6 @@ func initConfig() {
terminal.EnableColorsStdout()
}
- // Load filters
- err := filterflags.Reload(ctx)
- if err != nil {
- log.Fatalf("Failed to load filters: %v", err)
- }
-
// Write the args for debug purposes
fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
diff --git a/fs/configmap.go b/fs/configmap.go
index 9a0610c8e..4155f830d 100644
--- a/fs/configmap.go
+++ b/fs/configmap.go
@@ -33,10 +33,15 @@ func (oev optionEnvVars) Get(key string) (value string, ok bool) {
if opt == nil {
return "", false
}
- envKey := OptionToEnv(oev.prefix + "-" + key)
+ var envKey string
+ if oev.prefix == "" {
+ envKey = OptionToEnv(key)
+ } else {
+ envKey = OptionToEnv(oev.prefix + "-" + key)
+ }
value, ok = os.LookupEnv(envKey)
if ok {
- Debugf(nil, "Setting %s_%s=%q from environment variable %s", oev.prefix, key, value, envKey)
+ Debugf(nil, "Setting %s %s=%q from environment variable %s", oev.prefix, key, value, envKey)
} else if opt.NoPrefix {
// For options with NoPrefix set, check without prefix too
envKey := OptionToEnv(key)
@@ -115,10 +120,12 @@ func ConfigMap(prefix string, options Options, configName string, connectionStri
}
// remote specific environment vars
- config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)
+ if configName != "" {
+ config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)
+ }
// backend specific environment vars
- if options != nil && prefix != "" {
+ if options != nil {
config.AddGetter(optionEnvVars{prefix: prefix, options: options}, configmap.PriorityNormal)
}
diff --git a/fs/rc/config.go b/fs/rc/config.go
index f0cc4503a..1f538472d 100644
--- a/fs/rc/config.go
+++ b/fs/rc/config.go
@@ -12,21 +12,17 @@ import (
"github.com/rclone/rclone/fs/filter"
)
-var (
- optionBlock = map[string]interface{}{}
- optionReload = map[string]func(context.Context) error{}
-)
-
// AddOption adds an option set
func AddOption(name string, option interface{}) {
- optionBlock[name] = option
+ // FIXME remove this function when conversion to options is complete
+ fs.RegisterGlobalOptions(fs.OptionsInfo{Name: name, Opt: option})
}
// AddOptionReload adds an option set with a reload function to be
// called when options are changed
func AddOptionReload(name string, option interface{}, reload func(context.Context) error) {
- optionBlock[name] = option
- optionReload[name] = reload
+ // FIXME remove this function when conversion to options is complete
+ fs.RegisterGlobalOptions(fs.OptionsInfo{Name: name, Opt: option, Reload: reload})
}
func init() {
@@ -42,8 +38,8 @@ func init() {
// Show the list of all the option blocks
func rcOptionsBlocks(ctx context.Context, in Params) (out Params, err error) {
options := []string{}
- for name := range optionBlock {
- options = append(options, name)
+ for _, opt := range fs.OptionsRegistry {
+ options = append(options, opt.Name)
}
out = make(Params)
out["options"] = options
@@ -71,8 +67,8 @@ map to the external options very easily with a few exceptions.
// Show the list of all the option blocks
func rcOptionsGet(ctx context.Context, in Params) (out Params, err error) {
out = make(Params)
- for name, options := range optionBlock {
- out[name] = options
+ for _, opt := range fs.OptionsRegistry {
+ out[opt.Name] = opt.Opt
}
return out, nil
}
@@ -144,16 +140,16 @@ And this sets NOTICE level logs (normal without -v)
// Set an option in an option block
func rcOptionsSet(ctx context.Context, in Params) (out Params, err error) {
for name, options := range in {
- current := optionBlock[name]
- if current == nil {
+ opt, ok := fs.OptionsRegistry[name]
+ if !ok {
return nil, fmt.Errorf("unknown option block %q", name)
}
- err := Reshape(current, options)
+ err := Reshape(opt.Opt, options)
if err != nil {
return nil, fmt.Errorf("failed to write options from block %q: %w", name, err)
}
- if reload := optionReload[name]; reload != nil {
- err = reload(ctx)
+ if opt.Reload != nil {
+ err = opt.Reload(ctx)
if err != nil {
return nil, fmt.Errorf("failed to reload options from block %q: %w", name, err)
}
diff --git a/fs/rc/config_test.go b/fs/rc/config_test.go
index f3c75d164..b046f24bd 100644
--- a/fs/rc/config_test.go
+++ b/fs/rc/config_test.go
@@ -13,10 +13,10 @@ import (
)
func clearOptionBlock() func() {
- oldOptionBlock := optionBlock
- optionBlock = map[string]interface{}{}
+ oldOptionBlock := fs.OptionsRegistry
+ fs.OptionsRegistry = map[string]fs.OptionsInfo{}
return func() {
- optionBlock = oldOptionBlock
+ fs.OptionsRegistry = oldOptionBlock
}
}
@@ -30,22 +30,20 @@ var testOptions = struct {
func TestAddOption(t *testing.T) {
defer clearOptionBlock()()
- assert.Equal(t, len(optionBlock), 0)
+ assert.Equal(t, len(fs.OptionsRegistry), 0)
AddOption("potato", &testOptions)
- assert.Equal(t, len(optionBlock), 1)
- assert.Equal(t, len(optionReload), 0)
- assert.Equal(t, &testOptions, optionBlock["potato"])
+ assert.Equal(t, len(fs.OptionsRegistry), 1)
+ assert.Equal(t, &testOptions, fs.OptionsRegistry["potato"].Opt)
}
func TestAddOptionReload(t *testing.T) {
defer clearOptionBlock()()
- assert.Equal(t, len(optionBlock), 0)
+ assert.Equal(t, len(fs.OptionsRegistry), 0)
reload := func(ctx context.Context) error { return nil }
AddOptionReload("potato", &testOptions, reload)
- assert.Equal(t, len(optionBlock), 1)
- assert.Equal(t, len(optionReload), 1)
- assert.Equal(t, &testOptions, optionBlock["potato"])
- assert.Equal(t, fmt.Sprintf("%p", reload), fmt.Sprintf("%p", optionReload["potato"]))
+ assert.Equal(t, len(fs.OptionsRegistry), 1)
+ assert.Equal(t, &testOptions, fs.OptionsRegistry["potato"].Opt)
+ assert.Equal(t, fmt.Sprintf("%p", reload), fmt.Sprintf("%p", fs.OptionsRegistry["potato"].Reload))
}
func TestOptionsBlocks(t *testing.T) {
diff --git a/fs/registry.go b/fs/registry.go
index 2257623df..44f1606c1 100644
--- a/fs/registry.go
+++ b/fs/registry.go
@@ -61,6 +61,22 @@ func (ri *RegInfo) FileName() string {
// Options is a slice of configuration Option for a backend
type Options []Option
+// Add more options returning a new options slice
+func (os Options) Add(newOptions Options) Options {
+ return append(os, newOptions...)
+}
+
+// AddPrefix adds more options with a prefix returning a new options slice
+func (os Options) AddPrefix(newOptions Options, prefix string, groups string) Options {
+ for _, opt := range newOptions {
+ // opt is a copy so can modify
+ opt.Name = prefix + "_" + opt.Name
+ opt.Groups = groups
+ os = append(os, opt)
+ }
+ return os
+}
+
// Set the default values for the options
func (os Options) setValues() {
for i := range os {
@@ -91,6 +107,19 @@ func (os Options) Get(name string) *Option {
return nil
}
+// SetDefault sets the default for the Option corresponding to name
+//
+// Writes an ERROR level log if the option is not found
+func (os Options) SetDefault(name string, def any) Options {
+ opt := os.Get(name)
+ if opt == nil {
+ Errorf(nil, "Couldn't find option %q to SetDefault on", name)
+ } else {
+ opt.Default = def
+ }
+ return os
+}
+
// Overridden discovers which config items have been overridden in the
// configmap passed in, either by the config string, command line
// flags or environment variables
@@ -370,6 +399,62 @@ func MustFind(name string) *RegInfo {
return fs
}
+// OptionsInfo holds info about an block of options
+type OptionsInfo struct {
+ Name string // name of this options block for the rc
+ Opt interface{} // pointer to a struct to set the options in
+ Options Options // description of the options
+ Reload func(context.Context) error // if not nil, call when options changed and on init
+}
+
+// OptionsRegistry is a registry of global options
+var OptionsRegistry = map[string]OptionsInfo{}
+
+// RegisterGlobalOptions registers global options to be made into
+// command line options, rc options and environment variable reading.
+//
+// Packages which need global options should use this in an init() function
+func RegisterGlobalOptions(oi OptionsInfo) {
+ OptionsRegistry[oi.Name] = oi
+}
+
+// load the defaults from the options
+//
+// Reload the options if required
+func (oi *OptionsInfo) load() error {
+ if oi.Options == nil {
+ Errorf(nil, "No options defined for config block %q", oi.Name)
+ return nil
+ }
+
+ m := ConfigMap("", oi.Options, "", nil)
+ err := configstruct.Set(m, oi.Opt)
+ if err != nil {
+ return fmt.Errorf("failed to initialise %q options: %w", oi.Name, err)
+ }
+
+ if oi.Reload != nil {
+ err = oi.Reload(context.Background())
+ if err != nil {
+ return fmt.Errorf("failed to reload %q options: %w", oi.Name, err)
+ }
+ }
+ return nil
+}
+
+// GlobalOptionsInit initialises the defaults of global options to
+// their values read from the options, environment variables and
+// command line parameters.
+func GlobalOptionsInit() error {
+ for _, opt := range OptionsRegistry {
+ err := opt.load()
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
// Type returns a textual string to identify the type of the remote
func Type(f Fs) string {
typeName := fmt.Sprintf("%T", f)