"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
docker serve: parse all remaining mount and VFS options
Nick Craig-Wood33d2747829e1
diff --git a/cmd/serve/docker/options.go b/cmd/serve/docker/options.goindex c8c89a29c..e45820af7 100644--- a/cmd/serve/docker/options.go+++ b/cmd/serve/docker/options.go@@ -2,17 +2,15 @@ package dockerimport ("fmt"- "math""strings""github.com/rclone/rclone/cmd/mountlib""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/config/configstruct""github.com/rclone/rclone/fs/fspath""github.com/rclone/rclone/fs/rc""github.com/rclone/rclone/vfs/vfscommon"-- "github.com/spf13/pflag")// applyOptions configures volume from request options.@@ -112,11 +110,15 @@ func (vol *Volume) applyOptions(volOpt VolOpts) error {for key, val := range vol.Options {opt[key] = val}+ mntMap := configmap.Simple{}+ vfsMap := configmap.Simple{}for key := range opt {var ok boolvar err error+ normalKey := normalOptName(key)+ underscoreKey := strings.ReplaceAll(normalKey, "-", "_")- switch normalOptName(key) {+ switch normalKey {case "persist":vol.persist, err = opt.GetBool(key)ok = true@@ -129,25 +131,24 @@ func (vol *Volume) applyOptions(volOpt VolOpts) error {}if !ok {- // try to use as a mount option in mntOpt- ok, err = getMountOption(mntOpt, opt, key)- if ok && err != nil {- return fmt.Errorf("cannot parse mount option %q: %w", key, err)+ // try to use as a mount option in mntMap+ if mountlib.OptionsInfo.Get(underscoreKey) != nil {+ mntMap[underscoreKey] = vol.Options[key]+ ok = true}}if !ok {- // try as a vfs option in vfsOpt- ok, err = getVFSOption(vfsOpt, opt, key)- if ok && err != nil {- return fmt.Errorf("cannot parse vfs option %q: %w", key, err)+ // try as a vfs option in vfsMap+ if vfscommon.OptionsInfo.Get(underscoreKey) != nil {+ vfsMap[underscoreKey] = vol.Options[key]+ ok = true}}if !ok {// try as a backend option in fsOpt (backends use "_" instead of "-")- optWithPrefix := strings.ReplaceAll(normalOptName(key), "-", "_")- fsOptName := strings.TrimPrefix(optWithPrefix, fsType+"_")- hasFsPrefix := optWithPrefix != fsOptName+ fsOptName := strings.TrimPrefix(underscoreKey, fsType+"_")+ hasFsPrefix := underscoreKey != fsOptNameif !hasFsPrefix || fsInfo.Options.Get(fsOptName) == nil {fs.Logf(nil, "Option %q is not supported by backend %q", key, fsType)return fmt.Errorf("unsupported backend option %q", key)@@ -159,6 +160,18 @@ func (vol *Volume) applyOptions(volOpt VolOpts) error {}}+ // Parse VFS options+ err = configstruct.Set(vfsMap, vfsOpt)+ if err != nil {+ return fmt.Errorf("cannot parse vfs options: %w", err)+ }++ // Parse Mount options+ err = configstruct.Set(mntMap, mntOpt)+ if err != nil {+ return fmt.Errorf("cannot parse mount options: %w", err)+ }+// build remote string from fsName, fsType, fsOpt, fsPathcolon := ":"comma := ","@@ -178,150 +191,6 @@ func (vol *Volume) applyOptions(volOpt VolOpts) error {return vol.validate()}-func getMountOption(mntOpt *mountlib.Options, opt rc.Params, key string) (ok bool, err error) {- ok = true- switch normalOptName(key) {- case "debug-fuse":- mntOpt.DebugFUSE, err = opt.GetBool(key)- case "attr-timeout":- mntOpt.AttrTimeout, err = opt.GetFsDuration(key)- case "option":- mntOpt.ExtraOptions, err = getStringArray(opt, key)- case "fuse-flag":- mntOpt.ExtraFlags, err = getStringArray(opt, key)- case "daemon":- mntOpt.Daemon, err = opt.GetBool(key)- case "daemon-timeout":- mntOpt.DaemonTimeout, err = opt.GetFsDuration(key)- case "default-permissions":- mntOpt.DefaultPermissions, err = opt.GetBool(key)- case "allow-non-empty":- mntOpt.AllowNonEmpty, err = opt.GetBool(key)- case "allow-root":- mntOpt.AllowRoot, err = opt.GetBool(key)- case "allow-other":- mntOpt.AllowOther, err = opt.GetBool(key)- case "async-read":- mntOpt.AsyncRead, err = opt.GetBool(key)- case "max-read-ahead":- err = getFVarP(&mntOpt.MaxReadAhead, opt, key)- case "write-back-cache":- mntOpt.WritebackCache, err = opt.GetBool(key)- case "volname":- mntOpt.VolumeName, err = opt.GetString(key)- case "noappledouble":- mntOpt.NoAppleDouble, err = opt.GetBool(key)- case "noapplexattr":- mntOpt.NoAppleXattr, err = opt.GetBool(key)- case "network-mode":- mntOpt.NetworkMode, err = opt.GetBool(key)- default:- ok = false- }- return-}--func getVFSOption(vfsOpt *vfscommon.Options, opt rc.Params, key string) (ok bool, err error) {- var intVal int64- ok = true- switch normalOptName(key) {-- // options prefixed with "vfs-"- case "vfs-cache-mode":- err = getFVarP(&vfsOpt.CacheMode, opt, key)- case "vfs-cache-poll-interval":- vfsOpt.CachePollInterval, err = opt.GetFsDuration(key)- case "vfs-cache-max-age":- vfsOpt.CacheMaxAge, err = opt.GetFsDuration(key)- case "vfs-cache-max-size":- err = getFVarP(&vfsOpt.CacheMaxSize, opt, key)- case "vfs-read-chunk-size":- err = getFVarP(&vfsOpt.ChunkSize, opt, key)- case "vfs-read-chunk-size-limit":- err = getFVarP(&vfsOpt.ChunkSizeLimit, opt, key)- case "vfs-case-insensitive":- vfsOpt.CaseInsensitive, err = opt.GetBool(key)- case "vfs-write-wait":- vfsOpt.WriteWait, err = opt.GetFsDuration(key)- case "vfs-read-wait":- vfsOpt.ReadWait, err = opt.GetFsDuration(key)- case "vfs-write-back":- vfsOpt.WriteBack, err = opt.GetFsDuration(key)- case "vfs-read-ahead":- err = getFVarP(&vfsOpt.ReadAhead, opt, key)- case "vfs-used-is-size":- vfsOpt.UsedIsSize, err = opt.GetBool(key)- case "vfs-read-chunk-streams":- intVal, err = opt.GetInt64(key)- if err == nil {- if intVal >= 0 && intVal <= math.MaxInt {- vfsOpt.ChunkStreams = int(intVal)- } else {- err = fmt.Errorf("key %q (%v) overflows int", key, intVal)- }- }-- // unprefixed vfs options- case "no-modtime":- vfsOpt.NoModTime, err = opt.GetBool(key)- case "no-checksum":- vfsOpt.NoChecksum, err = opt.GetBool(key)- case "dir-cache-time":- vfsOpt.DirCacheTime, err = opt.GetFsDuration(key)- case "poll-interval":- vfsOpt.PollInterval, err = opt.GetFsDuration(key)- case "read-only":- vfsOpt.ReadOnly, err = opt.GetBool(key)- case "dir-perms":- err = getFVarP(&vfsOpt.DirPerms, opt, key)- case "file-perms":- err = getFVarP(&vfsOpt.FilePerms, opt, key)-- // unprefixed unix-only vfs options- case "umask":- err = getFVarP(&vfsOpt.Umask, opt, key)- case "uid":- intVal, err = opt.GetInt64(key)- if err == nil {- if intVal >= 0 && intVal <= math.MaxUint32 {- vfsOpt.UID = uint32(intVal)- } else {- err = fmt.Errorf("key %q (%v) overflows uint32", key, intVal)- }- }- case "gid":- intVal, err = opt.GetInt64(key)- if err == nil {- if intVal >= 0 && intVal <= math.MaxUint32 {- vfsOpt.GID = uint32(intVal)- } else {- err = fmt.Errorf("key %q (%v) overflows uint32", key, intVal)- }- }-- // non-vfs options- default:- ok = false- }- return-}--func getFVarP(pvalue pflag.Value, opt rc.Params, key string) error {- str, err := opt.GetString(key)- if err != nil {- return err- }- return pvalue.Set(str)-}--func getStringArray(opt rc.Params, key string) ([]string, error) {- str, err := opt.GetString(key)- if err != nil {- return nil, err- }- return strings.Split(str, ","), nil-}-func normalOptName(key string) string {return strings.ReplaceAll(strings.TrimPrefix(strings.ToLower(key), "--"), "_", "-")}diff --git a/cmd/serve/docker/options_test.go b/cmd/serve/docker/options_test.gonew file mode 100644index 000000000..f1d1d73dc--- /dev/null+++ b/cmd/serve/docker/options_test.go@@ -0,0 +1,75 @@+package docker++import (+ "testing"+ "time"++ "github.com/rclone/rclone/cmd/mountlib"+ "github.com/rclone/rclone/fs"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ _ "github.com/rclone/rclone/backend/local"+)++func TestApplyOptions(t *testing.T) {+ vol := &Volume{+ Name: "testName",+ MountPoint: "testPath",+ drv: &Driver{+ root: "testRoot",+ },+ mnt: &mountlib.MountPoint{+ MountPoint: "testPath",+ },+ mountReqs: make(map[string]interface{}),+ }++ // Happy path+ volOpt := VolOpts{+ "remote": "/tmp/docker",+ "persist": "FALSE",+ "mount_type": "potato",+ // backend options+ "--local-case-sensitive": "true",+ "local_no_check_updated": "1",+ // mount options+ "debug-fuse": "true",+ "attr_timeout": "100s",+ "--async-read": "TRUE",+ // vfs options+ "no-modtime": "1",+ "no_checksum": "true",+ "--no-seek": "true",+ }+ err := vol.applyOptions(volOpt)+ require.NoError(t, err)+ // normal options+ assert.Equal(t, ":local,case_sensitive='true',no_check_updated='1':/tmp/docker", vol.fsString)+ assert.Equal(t, false, vol.persist)+ assert.Equal(t, "potato", vol.mountType)+ // mount options+ assert.Equal(t, true, vol.mnt.MountOpt.DebugFUSE)+ assert.Equal(t, fs.Duration(100*time.Second), vol.mnt.MountOpt.AttrTimeout)+ assert.Equal(t, true, vol.mnt.MountOpt.AsyncRead)+ // vfs options+ assert.Equal(t, true, vol.mnt.VFSOpt.NoModTime)+ assert.Equal(t, true, vol.mnt.VFSOpt.NoChecksum)+ assert.Equal(t, true, vol.mnt.VFSOpt.NoSeek)++ // Check errors+ err = vol.applyOptions(VolOpts{+ "debug-fuse": "POTATO",+ })+ require.ErrorContains(t, err, "cannot parse mount options")+ err = vol.applyOptions(VolOpts{+ "no-modtime": "POTATO",+ })+ require.ErrorContains(t, err, "cannot parse vfs options")+ err = vol.applyOptions(VolOpts{+ "remote": "/tmp/docker",+ "local_not_found": "POTATO",+ })+ require.ErrorContains(t, err, "unsupported backend option")++}diff --git a/docs/content/docker.md b/docs/content/docker.mdindex 973df188b..319d24573 100644--- a/docs/content/docker.md+++ b/docs/content/docker.md@@ -209,7 +209,7 @@ but is arguably easier to parameterize in scripts.The `path` part is optional.[Mount and VFS options](/commands/rclone_serve_docker/#options)-as well as [backend parameters](/flags/#backend-flags) are named+as well as [backend parameters](/flags/#backend) are namedlike their twin command-line flags without the `--` CLI prefix.Optionally you can use underscores instead of dashes in option names.For example, `--vfs-cache-mode full` becomes