Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: add serve rc interface

Nick Craig-Woodaef9c2117ec1
diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go
index bef41c022..e71f11275 100644
--- a/cmd/serve/s3/s3.go
+++ b/cmd/serve/s3/s3.go
@@ -10,9 +10,12 @@ import (
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/flags"
+ "github.com/rclone/rclone/fs/rc"
httplib "github.com/rclone/rclone/lib/http"
"github.com/rclone/rclone/vfs"
+ "github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
)
@@ -61,6 +64,28 @@ func init() {
vfsflags.AddFlags(flagSet)
proxyflags.AddFlags(flagSet)
serve.Command.AddCommand(Command)
+ serve.AddRc("s3", func(ctx context.Context, f fs.Fs, in rc.Params) (serve.Handle, error) {
+ // Read VFS Opts
+ var vfsOpt = vfscommon.Opt // set default opts
+ err := configstruct.SetAny(in, &vfsOpt)
+ if err != nil {
+ return nil, err
+ }
+ // Read Proxy Opts
+ var proxyOpt = proxy.Opt // set default opts
+ err = configstruct.SetAny(in, &proxyOpt)
+ if err != nil {
+ return nil, err
+ }
+ // Read opts
+ var opt = Opt // set default opts
+ err = configstruct.SetAny(in, &opt)
+ if err != nil {
+ return nil, err
+ }
+ // Create server
+ return newServer(ctx, f, &opt, &vfsOpt, &proxyOpt)
+ })
}
//go:embed serve_s3.md
@@ -91,18 +116,11 @@ var Command = &cobra.Command{
}
cmd.Run(false, false, command, func() error {
- s, err := newServer(context.Background(), f, &Opt)
- if err != nil {
- return err
- }
- router := s.server.Router()
- s.Bind(router)
- err = s.Serve()
+ s, err := newServer(context.Background(), f, &Opt, &vfscommon.Opt, &proxy.Opt)
if err != nil {
return err
}
- s.server.Wait()
- return nil
+ return s.Serve()
})
return nil
},
diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.go
index 6bc73085f..15b19d36d 100644
--- a/cmd/serve/s3/s3_test.go
+++ b/cmd/serve/s3/s3_test.go
@@ -24,8 +24,10 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/random"
+ "github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -35,17 +37,16 @@ const (
)
// Configure and serve the server
-func serveS3(f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {
+func serveS3(t *testing.T, f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {
keyid = random.String(16)
keysec = random.String(16)
opt := Opt // copy default options
opt.AuthKey = []string{fmt.Sprintf("%s,%s", keyid, keysec)}
opt.HTTP.ListenAddr = []string{endpoint}
- w, _ = newServer(context.Background(), f, &opt)
- router := w.server.Router()
-
- w.Bind(router)
- _ = w.Serve()
+ w, _ = newServer(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt)
+ go func() {
+ require.NoError(t, w.Serve())
+ }()
testURL = w.server.URLs()[0]
return
@@ -55,7 +56,7 @@ func serveS3(f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {
// s3 remote against it.
func TestS3(t *testing.T) {
start := func(f fs.Fs) (configmap.Simple, func()) {
- testURL, keyid, keysec, _ := serveS3(f)
+ testURL, keyid, keysec, _ := serveS3(t, f)
// Config for the backend we'll use to connect to the server
config := configmap.Simple{
"type": "s3",
@@ -118,7 +119,7 @@ func TestEncodingWithMinioClient(t *testing.T) {
_, err = f.Put(context.Background(), in, obji)
assert.NoError(t, err)
- endpoint, keyid, keysec, _ := serveS3(f)
+ endpoint, keyid, keysec, _ := serveS3(t, f)
testURL, _ := url.Parse(endpoint)
minioClient, err := minio.New(testURL.Host, &minio.Options{
Creds: credentials.NewStaticV4(keyid, keysec, ""),
@@ -181,7 +182,7 @@ func testListBuckets(t *testing.T, cases []TestCase, useProxy bool) {
for _, tt := range cases {
t.Run(tt.description, func(t *testing.T) {
- endpoint, keyid, keysec, s := serveS3(f)
+ endpoint, keyid, keysec, s := serveS3(t, f)
defer func() {
assert.NoError(t, s.server.Shutdown())
}()
@@ -289,3 +290,10 @@ func TestListBucketsAuthProxy(t *testing.T) {
testListBuckets(t, cases, true)
}
+
+func TestRc(t *testing.T) {
+ servetest.TestRc(t, rc.Params{
+ "type": "s3",
+ "vfs_cache_mode": "off",
+ })
+}
diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go
index 05ee994ac..6732eb5e0 100644
--- a/cmd/serve/s3/server.go
+++ b/cmd/serve/s3/server.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"math/rand"
+ "net"
"net/http"
"strings"
@@ -43,7 +44,7 @@ type Server struct {
}
// Make a new S3 Server to serve the remote
-func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error) {
+func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Options, proxyOpt *proxy.Options) (s *Server, err error) {
w := &Server{
f: f,
ctx: ctx,
@@ -84,12 +85,12 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
w.handler = w.faker.Server()
if proxy.Opt.AuthProxy != "" {
- w.proxy = proxy.New(ctx, &proxy.Opt, &vfscommon.Opt)
+ w.proxy = proxy.New(ctx, proxyOpt, vfsOpt)
// proxy auth middleware
w.handler = proxyAuthMiddleware(w.handler, w)
w.handler = authPairMiddleware(w.handler, w)
} else {
- w._vfs = vfs.New(f, &vfscommon.Opt)
+ w._vfs = vfs.New(f, vfsOpt)
if len(opt.AuthKey) > 0 {
w.faker.AddAuthKeys(authlistResolver(opt.AuthKey))
@@ -104,6 +105,9 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
return nil, fmt.Errorf("failed to init server: %w", err)
}
+ router := w.server.Router()
+ w.Bind(router)
+
return w, nil
}
@@ -138,13 +142,24 @@ func (w *Server) Bind(router chi.Router) {
router.Handle("/*", w.handler)
}
-// Serve serves the s3 server
+// Serve serves the s3 server until the server is shutdown
func (w *Server) Serve() error {
w.server.Serve()
fs.Logf(w.f, "Starting s3 server on %s", w.server.URLs())
+ w.server.Wait()
return nil
}
+// Addr returns the first address of the server
+func (w *Server) Addr() net.Addr {
+ return w.server.Addr()
+}
+
+// Shutdown the server
+func (w *Server) Shutdown() error {
+ return w.server.Shutdown()
+}
+
func authPairMiddleware(next http.Handler, ws *Server) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
accessKey, _ := parseAccessKeyID(r)