Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

docker serve: parse all remaining mount and VFS options

Nick Craig-Wood33d2747829e1
diff --git a/cmd/serve/docker/options.go b/cmd/serve/docker/options.go
index c8c89a29c..e45820af7 100644
--- a/cmd/serve/docker/options.go
+++ b/cmd/serve/docker/options.go
@@ -2,17 +2,15 @@ package docker
import (
"fmt"
- "math"
"strings"
"github.com/rclone/rclone/cmd/mountlib"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fspath"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs/vfscommon"
-
- "github.com/spf13/pflag"
)
// applyOptions configures volume from request options.
@@ -112,11 +110,15 @@ func (vol *Volume) applyOptions(volOpt VolOpts) error {
for key, val := range vol.Options {
opt[key] = val
}
+ mntMap := configmap.Simple{}
+ vfsMap := configmap.Simple{}
for key := range opt {
var ok bool
var err error
+ normalKey := normalOptName(key)
+ underscoreKey := strings.ReplaceAll(normalKey, "-", "_")
- switch normalOptName(key) {
+ switch normalKey {
case "persist":
vol.persist, err = opt.GetBool(key)
ok = true
@@ -129,25 +131,24 @@ func (vol *Volume) applyOptions(volOpt VolOpts) error {
}
if !ok {
- // try to use as a mount option in mntOpt
- ok, err = getMountOption(mntOpt, opt, key)
- if ok && err != nil {
- return fmt.Errorf("cannot parse mount option %q: %w", key, err)
+ // try to use as a mount option in mntMap
+ if mountlib.OptionsInfo.Get(underscoreKey) != nil {
+ mntMap[underscoreKey] = vol.Options[key]
+ ok = true
}
}
if !ok {
- // try as a vfs option in vfsOpt
- ok, err = getVFSOption(vfsOpt, opt, key)
- if ok && err != nil {
- return fmt.Errorf("cannot parse vfs option %q: %w", key, err)
+ // try as a vfs option in vfsMap
+ if vfscommon.OptionsInfo.Get(underscoreKey) != nil {
+ vfsMap[underscoreKey] = vol.Options[key]
+ ok = true
}
}
if !ok {
// try as a backend option in fsOpt (backends use "_" instead of "-")
- optWithPrefix := strings.ReplaceAll(normalOptName(key), "-", "_")
- fsOptName := strings.TrimPrefix(optWithPrefix, fsType+"_")
- hasFsPrefix := optWithPrefix != fsOptName
+ fsOptName := strings.TrimPrefix(underscoreKey, fsType+"_")
+ hasFsPrefix := underscoreKey != fsOptName
if !hasFsPrefix || fsInfo.Options.Get(fsOptName) == nil {
fs.Logf(nil, "Option %q is not supported by backend %q", key, fsType)
return fmt.Errorf("unsupported backend option %q", key)
@@ -159,6 +160,18 @@ func (vol *Volume) applyOptions(volOpt VolOpts) error {
}
}
+ // Parse VFS options
+ err = configstruct.Set(vfsMap, vfsOpt)
+ if err != nil {
+ return fmt.Errorf("cannot parse vfs options: %w", err)
+ }
+
+ // Parse Mount options
+ err = configstruct.Set(mntMap, mntOpt)
+ if err != nil {
+ return fmt.Errorf("cannot parse mount options: %w", err)
+ }
+
// build remote string from fsName, fsType, fsOpt, fsPath
colon := ":"
comma := ","
@@ -178,150 +191,6 @@ func (vol *Volume) applyOptions(volOpt VolOpts) error {
return vol.validate()
}
-func getMountOption(mntOpt *mountlib.Options, opt rc.Params, key string) (ok bool, err error) {
- ok = true
- switch normalOptName(key) {
- case "debug-fuse":
- mntOpt.DebugFUSE, err = opt.GetBool(key)
- case "attr-timeout":
- mntOpt.AttrTimeout, err = opt.GetFsDuration(key)
- case "option":
- mntOpt.ExtraOptions, err = getStringArray(opt, key)
- case "fuse-flag":
- mntOpt.ExtraFlags, err = getStringArray(opt, key)
- case "daemon":
- mntOpt.Daemon, err = opt.GetBool(key)
- case "daemon-timeout":
- mntOpt.DaemonTimeout, err = opt.GetFsDuration(key)
- case "default-permissions":
- mntOpt.DefaultPermissions, err = opt.GetBool(key)
- case "allow-non-empty":
- mntOpt.AllowNonEmpty, err = opt.GetBool(key)
- case "allow-root":
- mntOpt.AllowRoot, err = opt.GetBool(key)
- case "allow-other":
- mntOpt.AllowOther, err = opt.GetBool(key)
- case "async-read":
- mntOpt.AsyncRead, err = opt.GetBool(key)
- case "max-read-ahead":
- err = getFVarP(&mntOpt.MaxReadAhead, opt, key)
- case "write-back-cache":
- mntOpt.WritebackCache, err = opt.GetBool(key)
- case "volname":
- mntOpt.VolumeName, err = opt.GetString(key)
- case "noappledouble":
- mntOpt.NoAppleDouble, err = opt.GetBool(key)
- case "noapplexattr":
- mntOpt.NoAppleXattr, err = opt.GetBool(key)
- case "network-mode":
- mntOpt.NetworkMode, err = opt.GetBool(key)
- default:
- ok = false
- }
- return
-}
-
-func getVFSOption(vfsOpt *vfscommon.Options, opt rc.Params, key string) (ok bool, err error) {
- var intVal int64
- ok = true
- switch normalOptName(key) {
-
- // options prefixed with "vfs-"
- case "vfs-cache-mode":
- err = getFVarP(&vfsOpt.CacheMode, opt, key)
- case "vfs-cache-poll-interval":
- vfsOpt.CachePollInterval, err = opt.GetFsDuration(key)
- case "vfs-cache-max-age":
- vfsOpt.CacheMaxAge, err = opt.GetFsDuration(key)
- case "vfs-cache-max-size":
- err = getFVarP(&vfsOpt.CacheMaxSize, opt, key)
- case "vfs-read-chunk-size":
- err = getFVarP(&vfsOpt.ChunkSize, opt, key)
- case "vfs-read-chunk-size-limit":
- err = getFVarP(&vfsOpt.ChunkSizeLimit, opt, key)
- case "vfs-case-insensitive":
- vfsOpt.CaseInsensitive, err = opt.GetBool(key)
- case "vfs-write-wait":
- vfsOpt.WriteWait, err = opt.GetFsDuration(key)
- case "vfs-read-wait":
- vfsOpt.ReadWait, err = opt.GetFsDuration(key)
- case "vfs-write-back":
- vfsOpt.WriteBack, err = opt.GetFsDuration(key)
- case "vfs-read-ahead":
- err = getFVarP(&vfsOpt.ReadAhead, opt, key)
- case "vfs-used-is-size":
- vfsOpt.UsedIsSize, err = opt.GetBool(key)
- case "vfs-read-chunk-streams":
- intVal, err = opt.GetInt64(key)
- if err == nil {
- if intVal >= 0 && intVal <= math.MaxInt {
- vfsOpt.ChunkStreams = int(intVal)
- } else {
- err = fmt.Errorf("key %q (%v) overflows int", key, intVal)
- }
- }
-
- // unprefixed vfs options
- case "no-modtime":
- vfsOpt.NoModTime, err = opt.GetBool(key)
- case "no-checksum":
- vfsOpt.NoChecksum, err = opt.GetBool(key)
- case "dir-cache-time":
- vfsOpt.DirCacheTime, err = opt.GetFsDuration(key)
- case "poll-interval":
- vfsOpt.PollInterval, err = opt.GetFsDuration(key)
- case "read-only":
- vfsOpt.ReadOnly, err = opt.GetBool(key)
- case "dir-perms":
- err = getFVarP(&vfsOpt.DirPerms, opt, key)
- case "file-perms":
- err = getFVarP(&vfsOpt.FilePerms, opt, key)
-
- // unprefixed unix-only vfs options
- case "umask":
- err = getFVarP(&vfsOpt.Umask, opt, key)
- case "uid":
- intVal, err = opt.GetInt64(key)
- if err == nil {
- if intVal >= 0 && intVal <= math.MaxUint32 {
- vfsOpt.UID = uint32(intVal)
- } else {
- err = fmt.Errorf("key %q (%v) overflows uint32", key, intVal)
- }
- }
- case "gid":
- intVal, err = opt.GetInt64(key)
- if err == nil {
- if intVal >= 0 && intVal <= math.MaxUint32 {
- vfsOpt.GID = uint32(intVal)
- } else {
- err = fmt.Errorf("key %q (%v) overflows uint32", key, intVal)
- }
- }
-
- // non-vfs options
- default:
- ok = false
- }
- return
-}
-
-func getFVarP(pvalue pflag.Value, opt rc.Params, key string) error {
- str, err := opt.GetString(key)
- if err != nil {
- return err
- }
- return pvalue.Set(str)
-}
-
-func getStringArray(opt rc.Params, key string) ([]string, error) {
- str, err := opt.GetString(key)
- if err != nil {
- return nil, err
- }
- return strings.Split(str, ","), nil
-}
-
func normalOptName(key string) string {
return strings.ReplaceAll(strings.TrimPrefix(strings.ToLower(key), "--"), "_", "-")
}
diff --git a/cmd/serve/docker/options_test.go b/cmd/serve/docker/options_test.go
new file mode 100644
index 000000000..f1d1d73dc
--- /dev/null
+++ b/cmd/serve/docker/options_test.go
@@ -0,0 +1,75 @@
+package docker
+
+import (
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/cmd/mountlib"
+ "github.com/rclone/rclone/fs"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ _ "github.com/rclone/rclone/backend/local"
+)
+
+func TestApplyOptions(t *testing.T) {
+ vol := &Volume{
+ Name: "testName",
+ MountPoint: "testPath",
+ drv: &Driver{
+ root: "testRoot",
+ },
+ mnt: &mountlib.MountPoint{
+ MountPoint: "testPath",
+ },
+ mountReqs: make(map[string]interface{}),
+ }
+
+ // Happy path
+ volOpt := VolOpts{
+ "remote": "/tmp/docker",
+ "persist": "FALSE",
+ "mount_type": "potato",
+ // backend options
+ "--local-case-sensitive": "true",
+ "local_no_check_updated": "1",
+ // mount options
+ "debug-fuse": "true",
+ "attr_timeout": "100s",
+ "--async-read": "TRUE",
+ // vfs options
+ "no-modtime": "1",
+ "no_checksum": "true",
+ "--no-seek": "true",
+ }
+ err := vol.applyOptions(volOpt)
+ require.NoError(t, err)
+ // normal options
+ assert.Equal(t, ":local,case_sensitive='true',no_check_updated='1':/tmp/docker", vol.fsString)
+ assert.Equal(t, false, vol.persist)
+ assert.Equal(t, "potato", vol.mountType)
+ // mount options
+ assert.Equal(t, true, vol.mnt.MountOpt.DebugFUSE)
+ assert.Equal(t, fs.Duration(100*time.Second), vol.mnt.MountOpt.AttrTimeout)
+ assert.Equal(t, true, vol.mnt.MountOpt.AsyncRead)
+ // vfs options
+ assert.Equal(t, true, vol.mnt.VFSOpt.NoModTime)
+ assert.Equal(t, true, vol.mnt.VFSOpt.NoChecksum)
+ assert.Equal(t, true, vol.mnt.VFSOpt.NoSeek)
+
+ // Check errors
+ err = vol.applyOptions(VolOpts{
+ "debug-fuse": "POTATO",
+ })
+ require.ErrorContains(t, err, "cannot parse mount options")
+ err = vol.applyOptions(VolOpts{
+ "no-modtime": "POTATO",
+ })
+ require.ErrorContains(t, err, "cannot parse vfs options")
+ err = vol.applyOptions(VolOpts{
+ "remote": "/tmp/docker",
+ "local_not_found": "POTATO",
+ })
+ require.ErrorContains(t, err, "unsupported backend option")
+
+}
diff --git a/docs/content/docker.md b/docs/content/docker.md
index 973df188b..319d24573 100644
--- a/docs/content/docker.md
+++ b/docs/content/docker.md
@@ -209,7 +209,7 @@ but is arguably easier to parameterize in scripts.
The `path` part is optional.
[Mount and VFS options](/commands/rclone_serve_docker/#options)
-as well as [backend parameters](/flags/#backend-flags) are named
+as well as [backend parameters](/flags/#backend) are named
like their twin command-line flags without the `--` CLI prefix.
Optionally you can use underscores instead of dashes in option names.
For example, `--vfs-cache-mode full` becomes