"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs: allow global variables to be overriden or set on backend creation
Nick Craig-Wood3c596f8d111f
diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 9996ff3fc..b247c2dc7 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -437,6 +437,103 @@ Do not use single character names on Windows as it creates ambiguity with Windowdrives' names, e.g.: remote called `C` is indistinguishable from `C` drive. Rclonewill always assume that single letter name refers to a drive.+## Adding global configuration to a remote++It is possible to add global configuration to the remote configuration which+will be applied just before the remote is created.++This can be done in two ways. The first is to use `override.var = value` in the+config file or the connection string for a temporary change, and the second is+to use `global.var = value` in the config file or connection string for a+permanent change.++This is explained fully below.++### override.var++This is used to override a global variable **just** for the duration of the+remote creation. It won't affect other remotes even if they are created at the+same time.++This is very useful for overriding networking config needed for just for that+remote. For example, say you have a remote which needs `--no-check-certificate`+as it is running on test infrastructure without a proper certificate. You could+supply the `--no-check-certificate` flag to rclone, but this will affect **all**+the remotes. To make it just affect this remote you use an override. You could+put this in the config file:++```ini+[remote]+type = XXX+...+override.no_check_certificate = true+```++or use it in the connection string `remote,override.no_check_certificate=true:`+(or just `remote,override.no_check_certificate:`).++Note how the global flag name loses its initial `--` and gets `-` replaced with+`_` and gets an `override.` prefix.++Not all global variables make sense to be overridden like this as the config is+only applied during the remote creation. Here is a non exhaustive list of ones+which might be useful:++- `bind_addr`+- `ca_cert`+- `client_cert`+- `client_key`+- `connect_timeout`+- `disable_http2`+- `disable_http_keep_alives`+- `dump`+- `expect_continue_timeout`+- `headers`+- `http_proxy`+- `low_level_retries`+- `max_connections`+- `no_check_certificate`+- `no_gzip`+- `timeout`+- `traffic_class`+- `use_cookies`+- `use_server_modtime`+- `user_agent`++An `override.var` will override all other config methods, but **just** for the+duration of the creation of the remote.++### global.var++This is used to set a global variable **for everything**. The global variable is+set just before the remote is created.++This is useful for parameters (eg sync parameters) which can't be set as an+`override`. For example, say you have a remote where you would always like to+use the `--checksum` flag. You could supply the `--checksum` flag to rclone on+every command line, but instead you could put this in the config file:++```ini+[remote]+type = XXX+...+global.checksum = true+```++or use it in the connection string `remote,global.checksum=true:` (or just+`remote,global.checksum:`). This is equivalent to using the `--checksum` flag.++Note how the global flag name loses its initial `--` and gets `-` replaced with+`_` and gets a `global.` prefix.++Any global variable can be set like this and it is exactly equivalent to using+the equivalent flag on the command line. This means it will affect all uses of+rclone.++If two remotes set the same global variable then the first one instantiated will+be overridden by the second one. A `global.var` will override all other config+methods when the remote is created.+## Quoting and the shellWhen you are typing commands to your computer you are using somethingdiff --git a/fs/fspath/path.go b/fs/fspath/path.goindex 09e71d08e..4cdd382c0 100644--- a/fs/fspath/path.go+++ b/fs/fspath/path.go@@ -20,7 +20,7 @@ const (var (errInvalidCharacters = errors.New("config name contains invalid characters - may only contain numbers, letters, `_`, `-`, `.`, `+`, `@` and space, while not start with `-` or space, and not end with space")errCantBeEmpty = errors.New("can't use empty string as a path")- errBadConfigParam = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z` and `_`")+ errBadConfigParam = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z`, `_` and `.`")errEmptyConfigParam = errors.New("config parameters can't be empty")errConfigNameEmpty = errors.New("config name can't be empty")errConfigName = errors.New("config name needs a trailing `:`")@@ -79,7 +79,8 @@ func isConfigParam(c rune) bool {return ((c >= 'a' && c <= 'z') ||(c >= 'A' && c <= 'Z') ||(c >= '0' && c <= '9') ||- c == '_')+ c == '_' ||+ c == '.')}// Parsed is returned from Parse with the results of the connection string decompositiondiff --git a/fs/newfs.go b/fs/newfs.goindex f4f7dd346..290e4ac40 100644--- a/fs/newfs.go+++ b/fs/newfs.go@@ -7,12 +7,15 @@ import ("crypto/md5""encoding/base64""fmt"+ "maps""os""path/filepath"+ "slices""strings""sync""github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/config/configstruct""github.com/rclone/rclone/fs/fspath")@@ -65,6 +68,10 @@ func NewFs(ctx context.Context, path string) (Fs, error) {overriddenConfig[suffix] = extraConfigoverriddenConfigMu.Unlock()}+ ctx, err = addConfigToContext(ctx, configName, config)+ if err != nil {+ return nil, err+ }f, err := fsInfo.NewFs(ctx, configName, fsPath, config)if f != nil && (err == nil || err == ErrorIsFile) {addReverse(f, fsInfo)@@ -72,6 +79,54 @@ func NewFs(ctx context.Context, path string) (Fs, error) {return f, err}+// Add "global" config or "override" to ctx and the global config if required.+//+// This looks through keys prefixed with "global." or "override." in+// config and sets ctx and optionally the global context if "global.".+func addConfigToContext(ctx context.Context, configName string, config configmap.Getter) (newCtx context.Context, err error) {+ overrideConfig := make(configmap.Simple)+ globalConfig := make(configmap.Simple)+ for i := range ConfigOptionsInfo {+ opt := &ConfigOptionsInfo[i]+ globalName := "global." + opt.Name+ value, isSet := config.Get(globalName)+ if isSet {+ // Set both override and global if global+ overrideConfig[opt.Name] = value+ globalConfig[opt.Name] = value+ }+ overrideName := "override." + opt.Name+ value, isSet = config.Get(overrideName)+ if isSet {+ overrideConfig[opt.Name] = value+ }+ }+ if len(overrideConfig) == 0 && len(globalConfig) == 0 {+ return ctx, nil+ }+ newCtx, ci := AddConfig(ctx)+ overrideKeys := slices.Collect(maps.Keys(overrideConfig))+ slices.Sort(overrideKeys)+ globalKeys := slices.Collect(maps.Keys(globalConfig))+ slices.Sort(globalKeys)+ // Set the config in the newCtx+ err = configstruct.Set(overrideConfig, ci)+ if err != nil {+ return ctx, fmt.Errorf("failed to set override config variables %q: %w", overrideKeys, err)+ }+ Debugf(configName, "Set overridden config %q for backend startup", overrideKeys)+ // Set the global context only+ if len(globalConfig) != 0 {+ globalCI := GetConfig(context.Background())+ err = configstruct.Set(globalConfig, globalCI)+ if err != nil {+ return ctx, fmt.Errorf("failed to set global config variables %q: %w", globalKeys, err)+ }+ Debugf(configName, "Set global config %q at backend startup", overrideKeys)+ }+ return newCtx, nil+}+// ConfigFs makes the config for calling NewFs with.//// It parses the path which is of the form remote:pathdiff --git a/fs/newfs_internal_test.go b/fs/newfs_internal_test.gonew file mode 100644index 000000000..6428a5bc8--- /dev/null+++ b/fs/newfs_internal_test.go@@ -0,0 +1,55 @@+package fs++import (+ "context"+ "testing"++ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// When no override/global keys exist, ctx must be returned unchanged.+func TestAddConfigToContext_NoChanges(t *testing.T) {+ ctx := context.Background()+ newCtx, err := addConfigToContext(ctx, "unit-test", configmap.Simple{})+ require.NoError(t, err)+ assert.Equal(t, newCtx, ctx)+}++// A single override.key must create a new ctx, but leave the+// background ctx untouched.+func TestAddConfigToContext_OverrideOnly(t *testing.T) {+ override := configmap.Simple{+ "override.user_agent": "potato",+ }+ ctx := context.Background()+ globalCI := GetConfig(ctx)+ original := globalCI.UserAgent+ newCtx, err := addConfigToContext(ctx, "unit-test", override)+ require.NoError(t, err)+ assert.NotEqual(t, newCtx, ctx)+ assert.Equal(t, original, globalCI.UserAgent)+ ci := GetConfig(newCtx)+ assert.Equal(t, "potato", ci.UserAgent)+}++// A single global.key must create a new ctx and update the+// background/global config.+func TestAddConfigToContext_GlobalOnly(t *testing.T) {+ global := configmap.Simple{+ "global.user_agent": "potato2",+ }+ ctx := context.Background()+ globalCI := GetConfig(ctx)+ original := globalCI.UserAgent+ defer func() {+ globalCI.UserAgent = original+ }()+ newCtx, err := addConfigToContext(ctx, "unit-test", global)+ require.NoError(t, err)+ assert.NotEqual(t, newCtx, ctx)+ assert.Equal(t, "potato2", globalCI.UserAgent)+ ci := GetConfig(newCtx)+ assert.Equal(t, "potato2", ci.UserAgent)+}diff --git a/fs/newfs_test.go b/fs/newfs_test.goindex 4d5ef8520..19667401e 100644--- a/fs/newfs_test.go+++ b/fs/newfs_test.go@@ -42,4 +42,21 @@ func TestNewFs(t *testing.T) {assert.Equal(t, ":mockfs{S_NHG}:/tmp", fs.ConfigString(f3))assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigStringFull(f3))++ // Check that the overrides work+ globalCI := fs.GetConfig(ctx)+ original := globalCI.UserAgent+ defer func() {+ globalCI.UserAgent = original+ }()++ f4, err := fs.NewFs(ctx, ":mockfs,global.user_agent='julian':/tmp")+ require.NoError(t, err)+ assert.Equal(t, ":mockfs", f4.Name())+ assert.Equal(t, "/tmp", f4.Root())++ assert.Equal(t, ":mockfs:/tmp", fs.ConfigString(f4))+ assert.Equal(t, ":mockfs:/tmp", fs.ConfigStringFull(f4))++ assert.Equal(t, "julian", globalCI.UserAgent)}