"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs: add Options registry and rework rc to use it
Nick Craig-Woode79273f9c984
diff --git a/cmd/cmd.go b/cmd/cmd.goindex 203253601..835033ef9 100644--- a/cmd/cmd.go+++ b/cmd/cmd.go@@ -29,7 +29,6 @@ import ("github.com/rclone/rclone/fs/config/configflags""github.com/rclone/rclone/fs/config/flags""github.com/rclone/rclone/fs/filter"- "github.com/rclone/rclone/fs/filter/filterflags""github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fspath"fslog "github.com/rclone/rclone/fs/log"@@ -383,6 +382,12 @@ func StartStats() func() {// initConfig is run by cobra after initialising the flagsfunc initConfig() {+ // Set the global options from the flags+ err := fs.GlobalOptionsInit()+ if err != nil {+ log.Fatalf("Failed to initialise global options: %v", err)+ }+ctx := context.Background()ci := fs.GetConfig(ctx)@@ -409,12 +414,6 @@ func initConfig() {terminal.EnableColorsStdout()}- // Load filters- err := filterflags.Reload(ctx)- if err != nil {- log.Fatalf("Failed to load filters: %v", err)- }-// Write the args for debug purposesfs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)diff --git a/fs/configmap.go b/fs/configmap.goindex 9a0610c8e..4155f830d 100644--- a/fs/configmap.go+++ b/fs/configmap.go@@ -33,10 +33,15 @@ func (oev optionEnvVars) Get(key string) (value string, ok bool) {if opt == nil {return "", false}- envKey := OptionToEnv(oev.prefix + "-" + key)+ var envKey string+ if oev.prefix == "" {+ envKey = OptionToEnv(key)+ } else {+ envKey = OptionToEnv(oev.prefix + "-" + key)+ }value, ok = os.LookupEnv(envKey)if ok {- Debugf(nil, "Setting %s_%s=%q from environment variable %s", oev.prefix, key, value, envKey)+ Debugf(nil, "Setting %s %s=%q from environment variable %s", oev.prefix, key, value, envKey)} else if opt.NoPrefix {// For options with NoPrefix set, check without prefix tooenvKey := OptionToEnv(key)@@ -115,10 +120,12 @@ func ConfigMap(prefix string, options Options, configName string, connectionStri}// remote specific environment vars- config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)+ if configName != "" {+ config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)+ }// backend specific environment vars- if options != nil && prefix != "" {+ if options != nil {config.AddGetter(optionEnvVars{prefix: prefix, options: options}, configmap.PriorityNormal)}diff --git a/fs/rc/config.go b/fs/rc/config.goindex f0cc4503a..1f538472d 100644--- a/fs/rc/config.go+++ b/fs/rc/config.go@@ -12,21 +12,17 @@ import ("github.com/rclone/rclone/fs/filter")-var (- optionBlock = map[string]interface{}{}- optionReload = map[string]func(context.Context) error{}-)-// AddOption adds an option setfunc AddOption(name string, option interface{}) {- optionBlock[name] = option+ // FIXME remove this function when conversion to options is complete+ fs.RegisterGlobalOptions(fs.OptionsInfo{Name: name, Opt: option})}// AddOptionReload adds an option set with a reload function to be// called when options are changedfunc AddOptionReload(name string, option interface{}, reload func(context.Context) error) {- optionBlock[name] = option- optionReload[name] = reload+ // FIXME remove this function when conversion to options is complete+ fs.RegisterGlobalOptions(fs.OptionsInfo{Name: name, Opt: option, Reload: reload})}func init() {@@ -42,8 +38,8 @@ func init() {// Show the list of all the option blocksfunc rcOptionsBlocks(ctx context.Context, in Params) (out Params, err error) {options := []string{}- for name := range optionBlock {- options = append(options, name)+ for _, opt := range fs.OptionsRegistry {+ options = append(options, opt.Name)}out = make(Params)out["options"] = options@@ -71,8 +67,8 @@ map to the external options very easily with a few exceptions.// Show the list of all the option blocksfunc rcOptionsGet(ctx context.Context, in Params) (out Params, err error) {out = make(Params)- for name, options := range optionBlock {- out[name] = options+ for _, opt := range fs.OptionsRegistry {+ out[opt.Name] = opt.Opt}return out, nil}@@ -144,16 +140,16 @@ And this sets NOTICE level logs (normal without -v)// Set an option in an option blockfunc rcOptionsSet(ctx context.Context, in Params) (out Params, err error) {for name, options := range in {- current := optionBlock[name]- if current == nil {+ opt, ok := fs.OptionsRegistry[name]+ if !ok {return nil, fmt.Errorf("unknown option block %q", name)}- err := Reshape(current, options)+ err := Reshape(opt.Opt, options)if err != nil {return nil, fmt.Errorf("failed to write options from block %q: %w", name, err)}- if reload := optionReload[name]; reload != nil {- err = reload(ctx)+ if opt.Reload != nil {+ err = opt.Reload(ctx)if err != nil {return nil, fmt.Errorf("failed to reload options from block %q: %w", name, err)}diff --git a/fs/rc/config_test.go b/fs/rc/config_test.goindex f3c75d164..b046f24bd 100644--- a/fs/rc/config_test.go+++ b/fs/rc/config_test.go@@ -13,10 +13,10 @@ import ()func clearOptionBlock() func() {- oldOptionBlock := optionBlock- optionBlock = map[string]interface{}{}+ oldOptionBlock := fs.OptionsRegistry+ fs.OptionsRegistry = map[string]fs.OptionsInfo{}return func() {- optionBlock = oldOptionBlock+ fs.OptionsRegistry = oldOptionBlock}}@@ -30,22 +30,20 @@ var testOptions = struct {func TestAddOption(t *testing.T) {defer clearOptionBlock()()- assert.Equal(t, len(optionBlock), 0)+ assert.Equal(t, len(fs.OptionsRegistry), 0)AddOption("potato", &testOptions)- assert.Equal(t, len(optionBlock), 1)- assert.Equal(t, len(optionReload), 0)- assert.Equal(t, &testOptions, optionBlock["potato"])+ assert.Equal(t, len(fs.OptionsRegistry), 1)+ assert.Equal(t, &testOptions, fs.OptionsRegistry["potato"].Opt)}func TestAddOptionReload(t *testing.T) {defer clearOptionBlock()()- assert.Equal(t, len(optionBlock), 0)+ assert.Equal(t, len(fs.OptionsRegistry), 0)reload := func(ctx context.Context) error { return nil }AddOptionReload("potato", &testOptions, reload)- assert.Equal(t, len(optionBlock), 1)- assert.Equal(t, len(optionReload), 1)- assert.Equal(t, &testOptions, optionBlock["potato"])- assert.Equal(t, fmt.Sprintf("%p", reload), fmt.Sprintf("%p", optionReload["potato"]))+ assert.Equal(t, len(fs.OptionsRegistry), 1)+ assert.Equal(t, &testOptions, fs.OptionsRegistry["potato"].Opt)+ assert.Equal(t, fmt.Sprintf("%p", reload), fmt.Sprintf("%p", fs.OptionsRegistry["potato"].Reload))}func TestOptionsBlocks(t *testing.T) {diff --git a/fs/registry.go b/fs/registry.goindex 2257623df..44f1606c1 100644--- a/fs/registry.go+++ b/fs/registry.go@@ -61,6 +61,22 @@ func (ri *RegInfo) FileName() string {// Options is a slice of configuration Option for a backendtype Options []Option+// Add more options returning a new options slice+func (os Options) Add(newOptions Options) Options {+ return append(os, newOptions...)+}++// AddPrefix adds more options with a prefix returning a new options slice+func (os Options) AddPrefix(newOptions Options, prefix string, groups string) Options {+ for _, opt := range newOptions {+ // opt is a copy so can modify+ opt.Name = prefix + "_" + opt.Name+ opt.Groups = groups+ os = append(os, opt)+ }+ return os+}+// Set the default values for the optionsfunc (os Options) setValues() {for i := range os {@@ -91,6 +107,19 @@ func (os Options) Get(name string) *Option {return nil}+// SetDefault sets the default for the Option corresponding to name+//+// Writes an ERROR level log if the option is not found+func (os Options) SetDefault(name string, def any) Options {+ opt := os.Get(name)+ if opt == nil {+ Errorf(nil, "Couldn't find option %q to SetDefault on", name)+ } else {+ opt.Default = def+ }+ return os+}+// Overridden discovers which config items have been overridden in the// configmap passed in, either by the config string, command line// flags or environment variables@@ -370,6 +399,62 @@ func MustFind(name string) *RegInfo {return fs}+// OptionsInfo holds info about an block of options+type OptionsInfo struct {+ Name string // name of this options block for the rc+ Opt interface{} // pointer to a struct to set the options in+ Options Options // description of the options+ Reload func(context.Context) error // if not nil, call when options changed and on init+}++// OptionsRegistry is a registry of global options+var OptionsRegistry = map[string]OptionsInfo{}++// RegisterGlobalOptions registers global options to be made into+// command line options, rc options and environment variable reading.+//+// Packages which need global options should use this in an init() function+func RegisterGlobalOptions(oi OptionsInfo) {+ OptionsRegistry[oi.Name] = oi+}++// load the defaults from the options+//+// Reload the options if required+func (oi *OptionsInfo) load() error {+ if oi.Options == nil {+ Errorf(nil, "No options defined for config block %q", oi.Name)+ return nil+ }++ m := ConfigMap("", oi.Options, "", nil)+ err := configstruct.Set(m, oi.Opt)+ if err != nil {+ return fmt.Errorf("failed to initialise %q options: %w", oi.Name, err)+ }++ if oi.Reload != nil {+ err = oi.Reload(context.Background())+ if err != nil {+ return fmt.Errorf("failed to reload %q options: %w", oi.Name, err)+ }+ }+ return nil+}++// GlobalOptionsInit initialises the defaults of global options to+// their values read from the options, environment variables and+// command line parameters.+func GlobalOptionsInit() error {+ for _, opt := range OptionsRegistry {+ err := opt.load()+ if err != nil {+ return err+ }+ }+ return nil+}+// Type returns a textual string to identify the type of the remotefunc Type(f Fs) string {typeName := fmt.Sprintf("%T", f)