Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fs: allow global variables to be overriden or set on backend creation

Nick Craig-Wood3c596f8d111f
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 9996ff3fc..b247c2dc7 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -437,6 +437,103 @@ Do not use single character names on Windows as it creates ambiguity with Window
drives' names, e.g.: remote called `C` is indistinguishable from `C` drive. Rclone
will always assume that single letter name refers to a drive.
+## Adding global configuration to a remote
+
+It is possible to add global configuration to the remote configuration which
+will be applied just before the remote is created.
+
+This can be done in two ways. The first is to use `override.var = value` in the
+config file or the connection string for a temporary change, and the second is
+to use `global.var = value` in the config file or connection string for a
+permanent change.
+
+This is explained fully below.
+
+### override.var
+
+This is used to override a global variable **just** for the duration of the
+remote creation. It won't affect other remotes even if they are created at the
+same time.
+
+This is very useful for overriding networking config needed for just for that
+remote. For example, say you have a remote which needs `--no-check-certificate`
+as it is running on test infrastructure without a proper certificate. You could
+supply the `--no-check-certificate` flag to rclone, but this will affect **all**
+the remotes. To make it just affect this remote you use an override. You could
+put this in the config file:
+
+```ini
+[remote]
+type = XXX
+...
+override.no_check_certificate = true
+```
+
+or use it in the connection string `remote,override.no_check_certificate=true:`
+(or just `remote,override.no_check_certificate:`).
+
+Note how the global flag name loses its initial `--` and gets `-` replaced with
+`_` and gets an `override.` prefix.
+
+Not all global variables make sense to be overridden like this as the config is
+only applied during the remote creation. Here is a non exhaustive list of ones
+which might be useful:
+
+- `bind_addr`
+- `ca_cert`
+- `client_cert`
+- `client_key`
+- `connect_timeout`
+- `disable_http2`
+- `disable_http_keep_alives`
+- `dump`
+- `expect_continue_timeout`
+- `headers`
+- `http_proxy`
+- `low_level_retries`
+- `max_connections`
+- `no_check_certificate`
+- `no_gzip`
+- `timeout`
+- `traffic_class`
+- `use_cookies`
+- `use_server_modtime`
+- `user_agent`
+
+An `override.var` will override all other config methods, but **just** for the
+duration of the creation of the remote.
+
+### global.var
+
+This is used to set a global variable **for everything**. The global variable is
+set just before the remote is created.
+
+This is useful for parameters (eg sync parameters) which can't be set as an
+`override`. For example, say you have a remote where you would always like to
+use the `--checksum` flag. You could supply the `--checksum` flag to rclone on
+every command line, but instead you could put this in the config file:
+
+```ini
+[remote]
+type = XXX
+...
+global.checksum = true
+```
+
+or use it in the connection string `remote,global.checksum=true:` (or just
+`remote,global.checksum:`). This is equivalent to using the `--checksum` flag.
+
+Note how the global flag name loses its initial `--` and gets `-` replaced with
+`_` and gets a `global.` prefix.
+
+Any global variable can be set like this and it is exactly equivalent to using
+the equivalent flag on the command line. This means it will affect all uses of
+rclone.
+
+If two remotes set the same global variable then the first one instantiated will
+be overridden by the second one. A `global.var` will override all other config
+methods when the remote is created.
+
## Quoting and the shell
When you are typing commands to your computer you are using something
diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index 09e71d08e..4cdd382c0 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -20,7 +20,7 @@ const (
var (
errInvalidCharacters = errors.New("config name contains invalid characters - may only contain numbers, letters, `_`, `-`, `.`, `+`, `@` and space, while not start with `-` or space, and not end with space")
errCantBeEmpty = errors.New("can't use empty string as a path")
- errBadConfigParam = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z` and `_`")
+ errBadConfigParam = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z`, `_` and `.`")
errEmptyConfigParam = errors.New("config parameters can't be empty")
errConfigNameEmpty = errors.New("config name can't be empty")
errConfigName = errors.New("config name needs a trailing `:`")
@@ -79,7 +79,8 @@ func isConfigParam(c rune) bool {
return ((c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
- c == '_')
+ c == '_' ||
+ c == '.')
}
// Parsed is returned from Parse with the results of the connection string decomposition
diff --git a/fs/newfs.go b/fs/newfs.go
index f4f7dd346..290e4ac40 100644
--- a/fs/newfs.go
+++ b/fs/newfs.go
@@ -7,12 +7,15 @@ import (
"crypto/md5"
"encoding/base64"
"fmt"
+ "maps"
"os"
"path/filepath"
+ "slices"
"strings"
"sync"
"github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fspath"
)
@@ -65,6 +68,10 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
overriddenConfig[suffix] = extraConfig
overriddenConfigMu.Unlock()
}
+ ctx, err = addConfigToContext(ctx, configName, config)
+ if err != nil {
+ return nil, err
+ }
f, err := fsInfo.NewFs(ctx, configName, fsPath, config)
if f != nil && (err == nil || err == ErrorIsFile) {
addReverse(f, fsInfo)
@@ -72,6 +79,54 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
return f, err
}
+// Add "global" config or "override" to ctx and the global config if required.
+//
+// This looks through keys prefixed with "global." or "override." in
+// config and sets ctx and optionally the global context if "global.".
+func addConfigToContext(ctx context.Context, configName string, config configmap.Getter) (newCtx context.Context, err error) {
+ overrideConfig := make(configmap.Simple)
+ globalConfig := make(configmap.Simple)
+ for i := range ConfigOptionsInfo {
+ opt := &ConfigOptionsInfo[i]
+ globalName := "global." + opt.Name
+ value, isSet := config.Get(globalName)
+ if isSet {
+ // Set both override and global if global
+ overrideConfig[opt.Name] = value
+ globalConfig[opt.Name] = value
+ }
+ overrideName := "override." + opt.Name
+ value, isSet = config.Get(overrideName)
+ if isSet {
+ overrideConfig[opt.Name] = value
+ }
+ }
+ if len(overrideConfig) == 0 && len(globalConfig) == 0 {
+ return ctx, nil
+ }
+ newCtx, ci := AddConfig(ctx)
+ overrideKeys := slices.Collect(maps.Keys(overrideConfig))
+ slices.Sort(overrideKeys)
+ globalKeys := slices.Collect(maps.Keys(globalConfig))
+ slices.Sort(globalKeys)
+ // Set the config in the newCtx
+ err = configstruct.Set(overrideConfig, ci)
+ if err != nil {
+ return ctx, fmt.Errorf("failed to set override config variables %q: %w", overrideKeys, err)
+ }
+ Debugf(configName, "Set overridden config %q for backend startup", overrideKeys)
+ // Set the global context only
+ if len(globalConfig) != 0 {
+ globalCI := GetConfig(context.Background())
+ err = configstruct.Set(globalConfig, globalCI)
+ if err != nil {
+ return ctx, fmt.Errorf("failed to set global config variables %q: %w", globalKeys, err)
+ }
+ Debugf(configName, "Set global config %q at backend startup", overrideKeys)
+ }
+ return newCtx, nil
+}
+
// ConfigFs makes the config for calling NewFs with.
//
// It parses the path which is of the form remote:path
diff --git a/fs/newfs_internal_test.go b/fs/newfs_internal_test.go
new file mode 100644
index 000000000..6428a5bc8
--- /dev/null
+++ b/fs/newfs_internal_test.go
@@ -0,0 +1,55 @@
+package fs
+
+import (
+ "context"
+ "testing"
+
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// When no override/global keys exist, ctx must be returned unchanged.
+func TestAddConfigToContext_NoChanges(t *testing.T) {
+ ctx := context.Background()
+ newCtx, err := addConfigToContext(ctx, "unit-test", configmap.Simple{})
+ require.NoError(t, err)
+ assert.Equal(t, newCtx, ctx)
+}
+
+// A single override.key must create a new ctx, but leave the
+// background ctx untouched.
+func TestAddConfigToContext_OverrideOnly(t *testing.T) {
+ override := configmap.Simple{
+ "override.user_agent": "potato",
+ }
+ ctx := context.Background()
+ globalCI := GetConfig(ctx)
+ original := globalCI.UserAgent
+ newCtx, err := addConfigToContext(ctx, "unit-test", override)
+ require.NoError(t, err)
+ assert.NotEqual(t, newCtx, ctx)
+ assert.Equal(t, original, globalCI.UserAgent)
+ ci := GetConfig(newCtx)
+ assert.Equal(t, "potato", ci.UserAgent)
+}
+
+// A single global.key must create a new ctx and update the
+// background/global config.
+func TestAddConfigToContext_GlobalOnly(t *testing.T) {
+ global := configmap.Simple{
+ "global.user_agent": "potato2",
+ }
+ ctx := context.Background()
+ globalCI := GetConfig(ctx)
+ original := globalCI.UserAgent
+ defer func() {
+ globalCI.UserAgent = original
+ }()
+ newCtx, err := addConfigToContext(ctx, "unit-test", global)
+ require.NoError(t, err)
+ assert.NotEqual(t, newCtx, ctx)
+ assert.Equal(t, "potato2", globalCI.UserAgent)
+ ci := GetConfig(newCtx)
+ assert.Equal(t, "potato2", ci.UserAgent)
+}
diff --git a/fs/newfs_test.go b/fs/newfs_test.go
index 4d5ef8520..19667401e 100644
--- a/fs/newfs_test.go
+++ b/fs/newfs_test.go
@@ -42,4 +42,21 @@ func TestNewFs(t *testing.T) {
assert.Equal(t, ":mockfs{S_NHG}:/tmp", fs.ConfigString(f3))
assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigStringFull(f3))
+
+ // Check that the overrides work
+ globalCI := fs.GetConfig(ctx)
+ original := globalCI.UserAgent
+ defer func() {
+ globalCI.UserAgent = original
+ }()
+
+ f4, err := fs.NewFs(ctx, ":mockfs,global.user_agent='julian':/tmp")
+ require.NoError(t, err)
+ assert.Equal(t, ":mockfs", f4.Name())
+ assert.Equal(t, "/tmp", f4.Root())
+
+ assert.Equal(t, ":mockfs:/tmp", fs.ConfigString(f4))
+ assert.Equal(t, ":mockfs:/tmp", fs.ConfigStringFull(f4))
+
+ assert.Equal(t, "julian", globalCI.UserAgent)
}