"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs: fix setting stringArray config values from environment variables
Nick Craig-Wood874d66658ed3
diff --git a/cmdtest/environment_test.go b/cmdtest/environment_test.goindex 3f1ad8035..2c161bf49 100644--- a/cmdtest/environment_test.go+++ b/cmdtest/environment_test.go@@ -6,7 +6,9 @@ package cmdtestimport ("os"+ "regexp""runtime"+ "strings""testing""github.com/stretchr/testify/assert"@@ -344,4 +346,42 @@ func TestEnvironmentVariables(t *testing.T) {env = ""out, err = rcloneEnv(env, "version", "-vv", "--use-json-log")jsonLogOK()++ // Find all the File filter lines in out and return them+ parseFileFilters := func(out string) (extensions []string) {+ // Match: - (^|/)[^/]*\.jpg$+ find := regexp.MustCompile(`^- \(\^\|\/\)\[\^\/\]\*\\\.(.*?)\$$`)+ for _, line := range strings.Split(out, "\n") {+ if m := find.FindStringSubmatch(line); m != nil {+ extensions = append(extensions, m[1])+ }+ }+ return extensions+ }++ // Make sure that multiple valued (stringArray) environment variables are handled properly+ env = ``+ out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters", "--exclude", "*.gif", "--exclude", "*.tif")+ require.NoError(t, err)+ assert.Equal(t, []string{"gif", "tif"}, parseFileFilters(out))++ env = `RCLONE_EXCLUDE=*.jpg`+ out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters", "--exclude", "*.gif")+ require.NoError(t, err)+ assert.Equal(t, []string{"jpg", "gif"}, parseFileFilters(out))++ env = `RCLONE_EXCLUDE=*.jpg,*.png`+ out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters", "--exclude", "*.gif", "--exclude", "*.tif")+ require.NoError(t, err)+ assert.Equal(t, []string{"jpg", "png", "gif", "tif"}, parseFileFilters(out))++ env = `RCLONE_EXCLUDE="*.jpg","*.png"`+ out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters")+ require.NoError(t, err)+ assert.Equal(t, []string{"jpg", "png"}, parseFileFilters(out))++ env = `RCLONE_EXCLUDE="*.,,,","*.png"`+ out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters")+ require.NoError(t, err)+ assert.Equal(t, []string{",,,", "png"}, parseFileFilters(out))}diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 7f6e208e5..eed34ed21 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -2911,6 +2911,22 @@ so they take exactly the same form.The options set by environment variables can be seen with the `-vv` flag, e.g. `rclone version -vv`.+Options that can appear multiple times (type `stringArray`) are+treated slighly differently as environment variables can only be+defined once. In order to allow a simple mechanism for adding one or+many items, the input is treated as a [CSV encoded](https://godoc.org/encoding/csv)+string. For example++| Environment Variable | Equivalent options |+|----------------------|--------------------|+| `RCLONE_EXCLUDE="*.jpg"` | `--exclude "*.jpg"` |+| `RCLONE_EXCLUDE="*.jpg,*.png"` | `--exclude "*.jpg"` `--exclude "*.png"` |+| `RCLONE_EXCLUDE='"*.jpg","*.png"'` | `--exclude "*.jpg"` `--exclude "*.png"` |+| `RCLONE_EXCLUDE='"/directory with comma , in it /**"'` | `--exclude "/directory with comma , in it /**" |++If `stringArray` options are defined as environment variables **and**+options on the command line then all the values will be used.+### Config file ###You can set defaults for values in the config file on an individualdiff --git a/fs/config/configstruct/configstruct.go b/fs/config/configstruct/configstruct.goindex 4d8c941f9..78322af9a 100644--- a/fs/config/configstruct/configstruct.go+++ b/fs/config/configstruct/configstruct.go@@ -2,7 +2,7 @@package configstructimport (- "encoding/json"+ "encoding/csv""errors""fmt""reflect"@@ -31,7 +31,7 @@ func camelToSnake(in string) string {//// Builtin types are expected to be encoding as their natural// stringificatons as produced by fmt.Sprint except for []string which-// is expected to be encoded as JSON with empty array encoded as "".+// is expected to be encoded a a CSV with empty array encoded as "".//// Any other types are expected to be encoded by their String()// methods and decoded by their `Set(s string) error` methods.@@ -58,14 +58,18 @@ func StringToInterface(def interface{}, in string) (newValue interface{}, err ercase time.Duration:newValue, err = time.ParseDuration(in)case []string:- // JSON decode arrays of strings- if in != "" {- var out []string- err = json.Unmarshal([]byte(in), &out)- newValue = out- } else {- // Empty string we will treat as empty array+ // CSV decode arrays of strings - ideally we would use+ // fs.CommaSepList here but we can't as it would cause+ // a circular import.+ if len(in) == 0 {newValue = []string{}+ } else {+ r := csv.NewReader(strings.NewReader(in))+ newValue, err = r.Read()+ switch _err := err.(type) {+ case *csv.ParseError:+ err = _err.Err // remove line numbers from the error message+ }}default:// Try using a Set methoddiff --git a/fs/config/configstruct/configstruct_test.go b/fs/config/configstruct/configstruct_test.goindex a31d70222..fa30684be 100644--- a/fs/config/configstruct/configstruct_test.go+++ b/fs/config/configstruct/configstruct_test.go@@ -204,9 +204,11 @@ func TestStringToInterface(t *testing.T) {{"1m1s", fs.Duration(0), fs.Duration(61 * time.Second), ""},{"1potato", fs.Duration(0), nil, `parsing "1potato" as fs.Duration failed: parsing time "1potato" as "2006-01-02": cannot parse "1potato" as "2006"`},{``, []string{}, []string{}, ""},- {`[]`, []string(nil), []string{}, ""},- {`["hello"]`, []string{}, []string{"hello"}, ""},- {`["hello","world!"]`, []string(nil), []string{"hello", "world!"}, ""},+ {`""`, []string(nil), []string{""}, ""},+ {`hello`, []string{}, []string{"hello"}, ""},+ {`"hello"`, []string{}, []string{"hello"}, ""},+ {`hello,world!`, []string(nil), []string{"hello", "world!"}, ""},+ {`"hello","world!"`, []string(nil), []string{"hello", "world!"}, ""},{"1s", time.Duration(0), time.Second, ""},{"1m1s", time.Duration(0), 61 * time.Second, ""},{"1potato", time.Duration(0), nil, `parsing "1potato" as time.Duration failed: time: unknown unit "potato" in duration "1potato"`},diff --git a/fs/config/flags/flags.go b/fs/config/flags/flags.goindex ebbd55f2a..953a3d0fd 100644--- a/fs/config/flags/flags.go+++ b/fs/config/flags/flags.go@@ -143,12 +143,32 @@ func installFlag(flags *pflag.FlagSet, name string, groupsString string) {// Read default from environment if possibleenvKey := fs.OptionToEnv(name)if envValue, envFound := os.LookupEnv(envKey); envFound {- err := flags.Set(name, envValue)- if err != nil {- fs.Fatalf(nil, "Invalid value when setting --%s from environment variable %s=%q: %v", name, envKey, envValue, err)+ isStringArray := false+ opt, isOption := flag.Value.(*fs.Option)+ if isOption {+ _, isStringArray = opt.Default.([]string)+ }+ if isStringArray {+ // Treat stringArray differently, treating the environment variable as a CSV array+ var list fs.CommaSepList+ err := list.Set(envValue)+ if err != nil {+ fs.Fatalf(nil, "Invalid value when setting stringArray --%s from environment variable %s=%q: %v", name, envKey, envValue, err)+ }+ // Set both the Value (so items on the command line get added) and DefValue so the help is correct+ opt.Value = ([]string)(list)+ flag.DefValue = list.String()+ for _, v := range list {+ fs.Debugf(nil, "Setting --%s %q from environment variable %s=%q", name, v, envKey, envValue)+ }+ } else {+ err := flags.Set(name, envValue)+ if err != nil {+ fs.Fatalf(nil, "Invalid value when setting --%s from environment variable %s=%q: %v", name, envKey, envValue, err)+ }+ fs.Debugf(nil, "Setting --%s %q from environment variable %s=%q", name, flag.Value, envKey, envValue)+ flag.DefValue = envValue}- fs.Debugf(nil, "Setting --%s %q from environment variable %s=%q", name, flag.Value, envKey, envValue)- flag.DefValue = envValue}// Add flag to Group if it is a global flagdiff --git a/fs/registry.go b/fs/registry.goindex 9d46e2602..e341bd2d5 100644--- a/fs/registry.go+++ b/fs/registry.go@@ -264,14 +264,9 @@ func (o *Option) String() string {if len(stringArray) == 0 {return ""}- // Encode string arrays as JSON+ // Encode string arrays as CSV// The default Go encoding can't be decoded uniquely- buf, err := json.Marshal(stringArray)- if err != nil {- Errorf(nil, "Can't encode default value for %q key - ignoring: %v", o.Name, err)- return "[]"- }- return string(buf)+ return CommaSepList(stringArray).String()}return fmt.Sprint(v)}