Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve: update serve remote control to use ParseOptions

Hakan İSMAİL01495c8dedca
diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go
index d24200b75..24a2e705b 100644
--- a/cmd/serve/dlna/dlna.go
+++ b/cmd/serve/dlna/dlna.go
@@ -24,7 +24,6 @@ import (
"github.com/rclone/rclone/cmd/serve"
"github.com/rclone/rclone/cmd/serve/dlna/data"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/lib/systemd"
@@ -78,13 +77,13 @@ func init() {
serve.AddRc("dlna", func(ctx context.Context, f fs.Fs, in rc.Params) (serve.Handle, error) {
// Read VFS Opts
var vfsOpt = vfscommon.Opt // set default opts
- err := configstruct.SetAny(in, &vfsOpt)
+ err := rc.ParseOptions(in, "vfsOpt", &vfsOpt)
if err != nil {
return nil, err
}
// Read opts
var opt = Opt // set default opts
- err = configstruct.SetAny(in, &opt)
+ err = rc.ParseOptions(in, "opt", &opt)
if err != nil {
return nil, err
}
diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go
index 401477780..e5f574ae1 100644
--- a/cmd/serve/ftp/ftp.go
+++ b/cmd/serve/ftp/ftp.go
@@ -24,7 +24,6 @@ import (
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
- "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/log"
@@ -96,19 +95,19 @@ func init() {
serve.AddRc("ftp", func(ctx context.Context, f fs.Fs, in rc.Params) (serve.Handle, error) {
// Read VFS Opts
var vfsOpt = vfscommon.Opt // set default opts
- err := configstruct.SetAny(in, &vfsOpt)
+ err := rc.ParseOptions(in, "vfsOpt", &vfsOpt)
if err != nil {
return nil, err
}
// Read Proxy Opts
var proxyOpt = proxy.Opt // set default opts
- err = configstruct.SetAny(in, &proxyOpt)
+ err = rc.ParseOptions(in, "proxyOpt", &proxyOpt)
if err != nil {
return nil, err
}
// Read opts
var opt = Opt // set default opts
- err = configstruct.SetAny(in, &opt)
+ err = rc.ParseOptions(in, "opt", &opt)
if err != nil {
return nil, err
}
diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index f1caa61d6..8cda23900 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -22,7 +22,6 @@ import (
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
- "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
libhttp "github.com/rclone/rclone/lib/http"
@@ -82,19 +81,19 @@ func init() {
cmdserve.AddRc("http", func(ctx context.Context, f fs.Fs, in rc.Params) (cmdserve.Handle, error) {
// Read VFS Opts
var vfsOpt = vfscommon.Opt // set default opts
- err := configstruct.SetAny(in, &vfsOpt)
+ err := rc.ParseOptions(in, "vfsOpt", &vfsOpt)
if err != nil {
return nil, err
}
// Read Proxy Opts
var proxyOpt = proxy.Opt // set default opts
- err = configstruct.SetAny(in, &proxyOpt)
+ err = rc.ParseOptions(in, "proxyOpt", &proxyOpt)
if err != nil {
return nil, err
}
// Read opts
var opt = Opt // set default opts
- err = configstruct.SetAny(in, &opt)
+ err = rc.ParseOptions(in, "opt", &opt)
if err != nil {
return nil, err
}
diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.go
index a4ca663cc..20bb6eddb 100644
--- a/cmd/serve/nfs/nfs.go
+++ b/cmd/serve/nfs/nfs.go
@@ -16,7 +16,6 @@ import (
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/serve"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs"
@@ -90,14 +89,14 @@ func init() {
serve.AddRc("nfs", func(ctx context.Context, f fs.Fs, in rc.Params) (serve.Handle, error) {
// Create VFS
var vfsOpt = vfscommon.Opt // set default opts
- err := configstruct.SetAny(in, &vfsOpt)
+ err := rc.ParseOptions(in, "vfsOpt", &vfsOpt)
if err != nil {
return nil, err
}
VFS := vfs.New(ctx, f, &vfsOpt)
// Read opts
var opt = Opt // set default opts
- err = configstruct.SetAny(in, &opt)
+ err = rc.ParseOptions(in, "opt", &opt)
if err != nil {
return nil, err
}
diff --git a/cmd/serve/rc.go b/cmd/serve/rc.go
index cbc6e7e7b..a0594102c 100644
--- a/cmd/serve/rc.go
+++ b/cmd/serve/rc.go
@@ -82,13 +82,15 @@ Other parameters are as described in the documentation for the
relevant [rclone serve](/commands/rclone_serve/) command line options.
To translate a command line option to an rc parameter, remove the
leading |--| and replace |-| with |_|, so |--vfs-cache-mode| becomes
-|vfs_cache_mode|. Note that global parameters must be set with
-|_config| and |_filter| as described above.
+|vfs_cache_mode|.
+
+Option parameters (such as VFS, proxy, and protocol-specific options) can be passed flat at the top level of the parameter map or inside nested JSON objects under the |vfsOpt|, |proxyOpt|, and |opt| keys (e.g. |vfsOpt='{"CacheMode": 2}'|, |proxyOpt='{"AuthProxy": "http://127.0.0.1:8080"}'|). If both flat parameters and nested blocks are supplied, the parameters in the nested blocks will take precedence. Note that global parameters must be set with |_config| and |_filter| as described above.
Examples:
rclone rc serve/start type=nfs fs=remote: addr=:4321 vfs_cache_mode=full
rclone rc serve/start --json '{"type":"nfs","fs":"remote:","addr":":1234","vfs_cache_mode":"full"}'
+ rclone rc serve/start type=webdav fs=remote: vfsOpt='{"CacheMode": 2}' proxyOpt='{"AuthProxy": "http://127.0.0.1:8080"}'
This will give the reply
@@ -108,6 +110,9 @@ Stop the server with |serve/stop| and list the running servers with |serve/list|
// startRc allows the serve command to be run from rc
func startRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ // Make a copy of input parameters to store in runningServer before parsing/deletion
+ paramsCopy := in.Copy()
+
serveType, err := in.GetString("type")
serveMu.Lock()
@@ -135,6 +140,14 @@ func startRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
return nil, fmt.Errorf("could not start serve %q: %w", serveType, err)
}
+ delete(in, "type")
+ delete(in, "fs")
+ err = rc.CheckParamsUsed(in)
+ if err != nil {
+ _ = h.Shutdown()
+ return nil, err
+ }
+
// Start the server running in the background
errChan := make(chan error, 1)
go func() {
@@ -158,7 +171,7 @@ func startRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
// Store it for later
runningServer := server{
ID: fmt.Sprintf("%s-%08x", serveType, rand.Uint32()),
- Params: in,
+ Params: paramsCopy,
Addr: h.Addr().String(),
h: h,
errChan: errChan,
diff --git a/cmd/serve/rc_test.go b/cmd/serve/rc_test.go
index e432ebfce..3650fbe2b 100644
--- a/cmd/serve/rc_test.go
+++ b/cmd/serve/rc_test.go
@@ -178,3 +178,65 @@ func TestRcStopAll(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, 0, len(servers))
}
+
+type mockServeOptions struct {
+ StringOpt string `config:"string_opt"`
+ IntOpt int `config:"int_opt"`
+}
+
+func newMockServer(ctx context.Context, f fs.Fs, in rc.Params) (Handle, error) {
+ var opt mockServeOptions
+ err := rc.ParseOptions(in, "serveOpt", &opt)
+ if err != nil {
+ return nil, err
+ }
+ return &dummyServer{
+ addr: &net.TCPAddr{
+ IP: net.IPv4(127, 0, 0, 1),
+ Port: 8080,
+ },
+ shutdownCh: make(chan struct{}),
+ }, nil
+}
+
+func TestRcStartFlatNestedAndUnknownRejection(t *testing.T) {
+ newTest(t)
+ serveStart := rc.Calls.Get("serve/start")
+ serveStop := rc.Calls.Get("serve/stop")
+
+ AddRc("mockserve", newMockServer)
+
+ t.Run("FlatAndNested", func(t *testing.T) {
+ in := rc.Params{
+ "fs": ":mockfs:",
+ "type": "mockserve",
+ "string_opt": "flat",
+ "serveOpt": rc.Params{
+ "IntOpt": 42,
+ },
+ }
+ out, err := serveStart.Fn(context.Background(), in)
+ require.NoError(t, err)
+ id := out["id"].(string)
+
+ // Verify the running server holds a copy of the original parameters
+ s := servers[id]
+ require.NotNil(t, s)
+ assert.Equal(t, "flat", s.Params["string_opt"])
+ assert.Equal(t, rc.Params{"IntOpt": 42}, s.Params["serveOpt"])
+
+ _, err = serveStop.Fn(context.Background(), rc.Params{"id": id})
+ require.NoError(t, err)
+ })
+
+ t.Run("UnknownRejection", func(t *testing.T) {
+ in := rc.Params{
+ "fs": ":mockfs:",
+ "type": "mockserve",
+ "unknown_param": "leftover",
+ }
+ _, err := serveStart.Fn(context.Background(), in)
+ assert.Error(t, err)
+ assert.ErrorContains(t, err, "unknown parameters: unknown_param")
+ })
+}
diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go
index 8c173a436..db7c16586 100644
--- a/cmd/serve/restic/restic.go
+++ b/cmd/serve/restic/restic.go
@@ -20,7 +20,6 @@ import (
cmdserve "github.com/rclone/rclone/cmd/serve"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
- "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/rc"
@@ -79,7 +78,7 @@ func init() {
cmdserve.AddRc("restic", func(ctx context.Context, f fs.Fs, in rc.Params) (cmdserve.Handle, error) {
// Read opts
var opt = Opt // set default opts
- err := configstruct.SetAny(in, &opt)
+ err := rc.ParseOptions(in, "opt", &opt)
if err != nil {
return nil, err
}
diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go
index 5712b10ea..1b0c92e61 100644
--- a/cmd/serve/s3/s3.go
+++ b/cmd/serve/s3/s3.go
@@ -10,7 +10,6 @@ import (
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
httplib "github.com/rclone/rclone/lib/http"
@@ -77,19 +76,19 @@ func init() {
serve.AddRc("s3", func(ctx context.Context, f fs.Fs, in rc.Params) (serve.Handle, error) {
// Read VFS Opts
var vfsOpt = vfscommon.Opt // set default opts
- err := configstruct.SetAny(in, &vfsOpt)
+ err := rc.ParseOptions(in, "vfsOpt", &vfsOpt)
if err != nil {
return nil, err
}
// Read Proxy Opts
var proxyOpt = proxy.Opt // set default opts
- err = configstruct.SetAny(in, &proxyOpt)
+ err = rc.ParseOptions(in, "proxyOpt", &proxyOpt)
if err != nil {
return nil, err
}
// Read opts
var opt = Opt // set default opts
- err = configstruct.SetAny(in, &opt)
+ err = rc.ParseOptions(in, "opt", &opt)
if err != nil {
return nil, err
}
diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go
index ebd6a0de0..50f54f8ab 100644
--- a/cmd/serve/sftp/sftp.go
+++ b/cmd/serve/sftp/sftp.go
@@ -13,7 +13,6 @@ import (
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/lib/systemd"
@@ -86,19 +85,19 @@ func init() {
serve.AddRc("sftp", func(ctx context.Context, f fs.Fs, in rc.Params) (serve.Handle, error) {
// Read VFS Opts
var vfsOpt = vfscommon.Opt // set default opts
- err := configstruct.SetAny(in, &vfsOpt)
+ err := rc.ParseOptions(in, "vfsOpt", &vfsOpt)
if err != nil {
return nil, err
}
// Read Proxy Opts
var proxyOpt = proxy.Opt // set default opts
- err = configstruct.SetAny(in, &proxyOpt)
+ err = rc.ParseOptions(in, "proxyOpt", &proxyOpt)
if err != nil {
return nil, err
}
// Read opts
var opt = Opt // set default opts
- err = configstruct.SetAny(in, &opt)
+ err = rc.ParseOptions(in, "opt", &opt)
if err != nil {
return nil, err
}
diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go
index 541aeb2b0..0a473e389 100644
--- a/cmd/serve/webdav/webdav.go
+++ b/cmd/serve/webdav/webdav.go
@@ -22,7 +22,6 @@ import (
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/rc"
@@ -81,19 +80,19 @@ func init() {
cmdserve.AddRc("webdav", func(ctx context.Context, f fs.Fs, in rc.Params) (cmdserve.Handle, error) {
// Read VFS Opts
var vfsOpt = vfscommon.Opt // set default opts
- err := configstruct.SetAny(in, &vfsOpt)
+ err := rc.ParseOptions(in, "vfsOpt", &vfsOpt)
if err != nil {
return nil, err
}
// Read Proxy Opts
var proxyOpt = proxy.Opt // set default opts
- err = configstruct.SetAny(in, &proxyOpt)
+ err = rc.ParseOptions(in, "proxyOpt", &proxyOpt)
if err != nil {
return nil, err
}
// Read opts
var opt = Opt // set default opts
- err = configstruct.SetAny(in, &opt)
+ err = rc.ParseOptions(in, "opt", &opt)
if err != nil {
return nil, err
}