"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
rc: allow setting rc config and filter options as flat parameters
Hakan İSMAİLb36b67920286
diff --git a/docs/content/rc.md b/docs/content/rc.mdindex 85b4f9a04..49cb5e58f 100644--- a/docs/content/rc.md+++ b/docs/content/rc.md@@ -404,6 +404,8 @@ This shows:If you wish to set config (the equivalent of the global flags) for theduration of an rc call only then pass in the `_config` parameter.+Alternatively, you can pass config options flat at the top level of the parameter map. The option names are the same as their CLI flags without `--` and with `-` replaced by `_` (e.g. `transfers` instead of `Transfers` inside `_config`).+This should be in the same format as the `main` key returned by[options/get](#options-get).@@ -425,12 +427,26 @@ parameter, you would pass this parameter in your JSON blob."_config":{"CheckSum": true}```+Or pass it flat at the top level:++```json+"checksum": true+```+If using `rclone rc` this could be passed as```consolerclone rc sync/sync ... _config='{"CheckSum": true}'```+Or simply flat:++```console+rclone rc sync/sync ... checksum=true+```++If both flat parameters and `_config` are supplied, the parameters in the legacy `_config` block will take precedence.+Any config parameters you don't set will inherit the global defaultswhich were set with command line flags or environment variables.@@ -443,6 +459,13 @@ setting the equivalent of `--buffer-size` in string or integer format."_config":{"BufferSize": 44040192}```+Or flat:++```json+"buffer_size": "42M"+"buffer_size": 44040192+```+If you wish to check the `_config` assignment has worked properly thencalling `options/local` will show what the value got set to.@@ -451,6 +474,8 @@ calling `options/local` will show what the value got set to.If you wish to set filters for the duration of an rc call only thenpass in the `_filter` parameter.+Alternatively, you can pass filter options flat at the top level of the parameter map. The option names are the same as their CLI flags without `--` and with `-` replaced by `_` (e.g. `exclude` instead of `Exclude` inside `_filter`).+This should be in the same format as the `filter` key returned by[options/get](#options-get).@@ -477,12 +502,26 @@ you would pass this parameter in your JSON blob."_filter":{"MaxSize":"1M", "IncludeRule":["a","b"], "MaxAge":"42s"}```+Or pass them flat at the top level:++```json+"max_size":"1M", "include":["a","b"], "max_age":"42s"+```+If using `rclone rc` this could be passed as```consolerclone rc ... _filter='{"MaxSize":"1M", "IncludeRule":["a","b"], "MaxAge":"42s"}'```+Or simply flat:++```console+rclone rc ... max_size=1M include="a,b" max_age=42s+```++If both flat parameters and `_filter` are supplied, the parameters in the legacy `_filter` block will take precedence.+Any filter parameters you don't set will inherit the global defaultswhich were set with command line flags or environment variables.@@ -495,6 +534,12 @@ setting the equivalent of `--buffer-size` in string or integer format."_filter":{"MinSize": 44040192}```+Or flat:++```json+"min_size": "42M"+```+If you wish to check the `_filter` assignment has worked properly thencalling `options/local` will show what the value got set to.diff --git a/fs/config/configstruct/configstruct.go b/fs/config/configstruct/configstruct.goindex 545f7d166..4fa95fe1c 100644--- a/fs/config/configstruct/configstruct.go+++ b/fs/config/configstruct/configstruct.go@@ -133,6 +133,19 @@ func InterfaceToString(in any) (strValue string, err error) {if do, ok := in.(fmt.Stringer); ok {strValue = do.String()} else {+ // Check if it is a slice or array+ val := reflect.ValueOf(in)+ if val.Kind() == reflect.Slice || val.Kind() == reflect.Array {+ strSlice := make([]string, val.Len())+ for i := 0; i < val.Len(); i++ {+ strValue, err = InterfaceToString(val.Index(i).Interface())+ if err != nil {+ return "", err+ }+ strSlice[i] = strValue+ }+ return InterfaceToString(strSlice)+ }err = errors.New("don't know how to convert this")}}@@ -262,14 +275,35 @@ func Set(config configmap.Getter, opt any) (err error) {// setIfSameType set aPtr with b if they are the same type or returns false.func setIfSameType(aPtr any, b any) bool {+ if b == nil {+ return false+ }aVal := reflect.ValueOf(aPtr).Elem()bVal := reflect.ValueOf(b)- if aVal.Type() != bVal.Type() {- return false+ if aVal.Type() == bVal.Type() {+ aVal.Set(bVal)+ return true+ }++ // Special case: if target is []string and source is a slice or array+ if aVal.Type() == reflect.TypeOf([]string(nil)) && (bVal.Kind() == reflect.Slice || bVal.Kind() == reflect.Array) {+ strSlice := make([]string, bVal.Len())+ ok := true+ for i := 0; i < bVal.Len(); i++ {+ var err error+ strSlice[i], err = InterfaceToString(bVal.Index(i).Interface())+ if err != nil {+ ok = false+ break+ }+ }+ if ok {+ aVal.Set(reflect.ValueOf(strSlice))+ return true+ }}- aVal.Set(bVal)- return true+ return false}// SetAny interprets the field names in defaults and looks up configdiff --git a/fs/config/configstruct/configstruct_test.go b/fs/config/configstruct/configstruct_test.goindex fb0337220..79ebffbfc 100644--- a/fs/config/configstruct/configstruct_test.go+++ b/fs/config/configstruct/configstruct_test.go@@ -209,6 +209,45 @@ func TestSetAnyFull(t *testing.T) {assert.Equal(t, want, in)}+func TestSetAnyWithSlice(t *testing.T) {+ type ConfWithSlice struct {+ ExcludeRule []string `config:"exclude"`+ }++ // 1. Test []any (JSON array of strings)+ in := &ConfWithSlice{+ ExcludeRule: []string{"original"},+ }+ m := map[string]any{+ "exclude": []any{"foo", "bar"},+ }+ err := configstruct.SetAny(m, in)+ require.NoError(t, err)+ assert.Equal(t, []string{"foo", "bar"}, in.ExcludeRule)++ // 2. Test []string directly+ in2 := &ConfWithSlice{+ ExcludeRule: []string{"original"},+ }+ m2 := map[string]any{+ "exclude": []string{"hello", "world"},+ }+ err = configstruct.SetAny(m2, in2)+ require.NoError(t, err)+ assert.Equal(t, []string{"hello", "world"}, in2.ExcludeRule)++ // 3. Test []any with mixed convertible types (e.g. ints/floats/bools)+ in3 := &ConfWithSlice{+ ExcludeRule: []string{"original"},+ }+ m3 := map[string]any{+ "exclude": []any{"foo", 123, true, 45.6},+ }+ err = configstruct.SetAny(m3, in3)+ require.NoError(t, err)+ assert.Equal(t, []string{"foo", "123", "true", "45.6"}, in3.ExcludeRule)+}+func TestStringToInterface(t *testing.T) {item := struct{ A int }{2}for _, test := range []struct {@@ -288,6 +327,13 @@ func TestInterfaceToString(t *testing.T) {{[]string{"hello", "world"}, `hello,world`, ""},{[]string{"hello", "", "world"}, `hello,,world`, ""},{[]string{`hello, world`, `goodbye, world!`}, `"hello, world","goodbye, world!"`, ""},+ {[]any{"hello", "world"}, `hello,world`, ""},+ {[]any{123, 456}, `123,456`, ""},+ {[]any{"hello", 45.6, true}, `hello,45.6,true`, ""},+ {[2]string{"a", "b"}, `a,b`, ""},+ {[2]any{"x", 12}, `x,12`, ""},+ {fs.SpaceSepList{"a", "b"}, `a b`, ""},+ {fs.CommaSepList{"x", "y"}, `x,y`, ""},{time.Second, "1s", ""},{61 * time.Second, "1m1s", ""},{fs.Mebi, "1Mi", ""},diff --git a/fs/rc/context.go b/fs/rc/context.gonew file mode 100644index 000000000..3f68c0c4f--- /dev/null+++ b/fs/rc/context.go@@ -0,0 +1,170 @@+package rc++import (+ "context"+ "reflect"+ "sync"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/filter"+)++var (+ configOptionsOnce sync.Once+ configOptionsMap map[string]bool++ filterOptionsOnce sync.Once+ filterOptionsMap map[string]bool+)++func initConfigOptions() {+ configOptionsOnce.Do(func() {+ configOptionsMap = make(map[string]bool, len(fs.ConfigOptionsInfo))+ for _, opt := range fs.ConfigOptionsInfo {+ configOptionsMap[opt.Name] = true+ }+ })+}++func initFilterOptions() {+ filterOptionsOnce.Do(func() {+ filterOptionsMap = make(map[string]bool, len(filter.OptionsInfo))+ for _, opt := range filter.OptionsInfo {+ filterOptionsMap[opt.Name] = true+ }+ })+}++// isMap returns true if v's underlying type is a map+func isMap(v any) bool {+ if v == nil {+ return false+ }+ t := reflect.TypeOf(v)+ for t.Kind() == reflect.Pointer {+ t = t.Elem()+ }+ return t.Kind() == reflect.Map+}++// hasConfigOption checks if any config options are present in the params+func hasConfigOption(in Params) bool {+ if _, ok := in["_config"]; ok {+ return true+ }+ initConfigOptions()+ for k, v := range in {+ if configOptionsMap[k] {+ if isMap(v) {+ continue+ }+ return true+ }+ }+ return false+}++// hasFilterOption checks if any filter options are present in the params+func hasFilterOption(in Params) bool {+ if _, ok := in["_filter"]; ok {+ return true+ }+ initFilterOptions()+ for k, v := range in {+ if filterOptionsMap[k] {+ if isMap(v) {+ continue+ }+ return true+ }+ }+ return false+}++// AddConfig parses any config options from the parameters and returns a new context with the configuration.+func AddConfig(ctx context.Context, in Params) (context.Context, error) {+ if !hasConfigOption(in) {+ return ctx, nil+ }+ ctx, ci := fs.AddConfig(ctx)++ // Extract the genuine flat config options+ initConfigOptions()+ flatConfig := make(map[string]any)+ for k, v := range in {+ if configOptionsMap[k] {+ if isMap(v) {+ continue+ }+ flatConfig[k] = v+ }+ }++ if len(flatConfig) > 0 {+ err := configstruct.SetAny(flatConfig, ci)+ if err != nil {+ return ctx, err+ }+ // Remove the consumed flat options from the input params+ for k := range flatConfig {+ delete(in, k)+ }+ }++ if _, ok := in["_config"]; ok {+ err := in.GetStruct("_config", ci)+ if err != nil {+ return ctx, err+ }+ delete(in, "_config") // remove the parameter+ }+ return ctx, nil+}++// AddFilter parses any filter options from the parameters and returns a new context with the filter.+func AddFilter(ctx context.Context, in Params) (context.Context, error) {+ if !hasFilterOption(in) {+ return ctx, nil+ }+ // Copy of the current filter options+ opt := filter.GetConfig(ctx).Opt++ // Extract the genuine flat filter options+ initFilterOptions()+ flatFilter := make(map[string]any)+ for k, v := range in {+ if filterOptionsMap[k] {+ if isMap(v) {+ continue+ }+ flatFilter[k] = v+ }+ }++ if len(flatFilter) > 0 {+ err := configstruct.SetAny(flatFilter, &opt)+ if err != nil {+ return ctx, err+ }+ // Remove the consumed flat options from the input params+ for k := range flatFilter {+ delete(in, k)+ }+ }++ if _, ok := in["_filter"]; ok {+ // Update the options from the parameter+ err := in.GetStruct("_filter", &opt)+ if err != nil {+ return ctx, err+ }+ delete(in, "_filter") // remove the parameter+ }+ fi, err := filter.NewFilter(&opt)+ if err != nil {+ return ctx, err+ }+ ctx = filter.ReplaceConfig(ctx, fi)+ return ctx, nil+}diff --git a/fs/rc/jobs/job.go b/fs/rc/jobs/job.goindex 2ba939210..e1acfae4c 100644--- a/fs/rc/jobs/job.go+++ b/fs/rc/jobs/job.go@@ -19,7 +19,6 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/cache"- "github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/rc""golang.org/x/sync/errgroup")@@ -242,41 +241,6 @@ func getAsync(ctx context.Context, in rc.Params) (context.Context, bool, error)return ctx, isAsync, nil}-// See if _config is set and if so adjust ctx to include it-func getConfig(ctx context.Context, in rc.Params) (context.Context, error) {- if _, ok := in["_config"]; !ok {- return ctx, nil- }- ctx, ci := fs.AddConfig(ctx)- err := in.GetStruct("_config", ci)- if err != nil {- return ctx, err- }- delete(in, "_config") // remove the parameter- return ctx, nil-}--// See if _filter is set and if so adjust ctx to include it-func getFilter(ctx context.Context, in rc.Params) (context.Context, error) {- if _, ok := in["_filter"]; !ok {- return ctx, nil- }- // Copy of the current filter options- opt := filter.GetConfig(ctx).Opt- // Update the options from the parameter- err := in.GetStruct("_filter", &opt)- if err != nil {- return ctx, err- }- fi, err := filter.NewFilter(&opt)- if err != nil {- return ctx, err- }- ctx = filter.ReplaceConfig(ctx, fi)- delete(in, "_filter") // remove the parameter- return ctx, nil-}-type jobKeyType struct{}// Key for adding jobs to ctx@@ -292,12 +256,12 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Joreturn nil, nil, err}- ctx, err = getConfig(ctx, in)+ ctx, err = rc.AddConfig(ctx, in)if err != nil {return nil, nil, err}- ctx, err = getFilter(ctx, in)+ ctx, err = rc.AddFilter(ctx, in)if err != nil {return nil, nil, err}diff --git a/fs/rc/jobs/job_test.go b/fs/rc/jobs/job_test.goindex eaf95aead..02f171860 100644--- a/fs/rc/jobs/job_test.go+++ b/fs/rc/jobs/job_test.go@@ -352,6 +352,141 @@ func TestExecuteJobWithFilter(t *testing.T) {assert.Equal(t, true, called)}+func TestExecuteJobWithFlatConfig(t *testing.T) {+ ctx := context.Background()+ jobID.Store(0)+ called := false+ jobFn := func(ctx context.Context, in rc.Params) (rc.Params, error) {+ ci := fs.GetConfig(ctx)+ assert.Equal(t, 42*fs.Mebi, ci.BufferSize)+ called = true+ return nil, nil+ }+ _, _, err := NewJob(ctx, jobFn, rc.Params{+ "buffer_size": "42M",+ })+ require.NoError(t, err)+ assert.Equal(t, true, called)++ // Test that legacy _config overrides flat parameter+ jobID.Store(0)+ called = false+ jobFn2 := func(ctx context.Context, in rc.Params) (rc.Params, error) {+ ci := fs.GetConfig(ctx)+ assert.Equal(t, 10*fs.Mebi, ci.BufferSize)+ called = true+ return nil, nil+ }+ _, _, err = NewJob(ctx, jobFn2, rc.Params{+ "buffer_size": "42M",+ "_config": rc.Params{+ "BufferSize": "10M",+ },+ })+ require.NoError(t, err)+ assert.Equal(t, true, called)+}++func TestExecuteJobWithFlatFilter(t *testing.T) {+ ctx := context.Background()+ called := false+ jobID.Store(0)+ jobFn := func(ctx context.Context, in rc.Params) (rc.Params, error) {+ fi := filter.GetConfig(ctx)+ assert.Equal(t, fs.SizeSuffix(1024), fi.Opt.MaxSize)+ assert.Equal(t, []string{"a", "b", "c"}, fi.Opt.IncludeRule)+ called = true+ return nil, nil+ }+ _, _, err := NewJob(ctx, jobFn, rc.Params{+ "include": []string{"a", "b", "c"},+ "max_size": "1k",+ })+ require.NoError(t, err)+ assert.Equal(t, true, called)++ // Test that legacy _filter overrides flat parameter+ called = false+ jobID.Store(0)+ jobFn2 := func(ctx context.Context, in rc.Params) (rc.Params, error) {+ fi := filter.GetConfig(ctx)+ assert.Equal(t, fs.SizeSuffix(2048), fi.Opt.MaxSize)+ assert.Equal(t, []string{"x", "y"}, fi.Opt.IncludeRule)+ called = true+ return nil, nil+ }+ _, _, err = NewJob(ctx, jobFn2, rc.Params{+ "include": []string{"a", "b", "c"},+ "max_size": "1k",+ "_filter": rc.Params{+ "IncludeRule": []string{"x", "y"},+ "MaxSize": "2k",+ },+ })+ require.NoError(t, err)+ assert.Equal(t, true, called)+}++// A null-valued flat config/filter param must produce a clean+// error, not panic the rc handler.+func TestExecuteJobWithFlatConfigNull(t *testing.T) {+ ctx := context.Background()+ jobID.Store(0)+ called := false+ jobFn := func(ctx context.Context, in rc.Params) (rc.Params, error) {+ called = true+ return nil, nil+ }+ var err error+ require.NotPanics(t, func() {+ _, _, err = NewJob(ctx, jobFn, rc.Params{+ "buffer_size": nil,+ })+ })+ assert.Error(t, err)+ assert.False(t, called)+}++// Flat config/filter options should be consumed and removed from+// the params (like _config and _filter are), so they don't leak into the+// command's parameter map.+func TestExecuteJobFlatParamsRemoved(t *testing.T) {+ ctx := context.Background()+ jobID.Store(0)+ var got rc.Params+ jobFn := func(ctx context.Context, in rc.Params) (rc.Params, error) {+ got = in+ return nil, nil+ }+ _, _, err := NewJob(ctx, jobFn, rc.Params{+ "buffer_size": "42M",+ "max_size": "1k",+ "include": []string{"a"},+ })+ require.NoError(t, err)+ _, ok := got["buffer_size"]+ assert.False(t, ok, "flat config option buffer_size should have been removed from in")+ _, ok = got["max_size"]+ assert.False(t, ok, "flat filter option max_size should have been removed from in")+ _, ok = got["include"]+ assert.False(t, ok, "flat filter option include should have been removed from in")+}++// options/set with a "filter" block is a valid, documented call, but+// the flat-parameter feature treats the top-level "filter" key (a registered+// filter option name) as the --filter option and fails trying to parse the+// block map as a string, breaking the call.+func TestExecuteJobOptionsSetFilterBlock(t *testing.T) {+ ctx := context.Background()+ jobID.Store(0)+ call := rc.Calls.Get("options/set")+ require.NotNil(t, call)+ _, _, err := NewJob(ctx, call.Fn, rc.Params{+ "filter": rc.Params{"MaxSize": "1M"},+ })+ require.NoError(t, err)+}+func TestExecuteJobWithGroup(t *testing.T) {ctx := context.Background()jobID.Store(0)