"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: add serve rc interface
Nick Craig-Woodaef9c2117ec1
diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.goindex bef41c022..e71f11275 100644--- a/cmd/serve/s3/s3.go+++ b/cmd/serve/s3/s3.go@@ -10,9 +10,12 @@ import ("github.com/rclone/rclone/cmd/serve/proxy""github.com/rclone/rclone/cmd/serve/proxy/proxyflags""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/configstruct""github.com/rclone/rclone/fs/config/flags"+ "github.com/rclone/rclone/fs/rc"httplib "github.com/rclone/rclone/lib/http""github.com/rclone/rclone/vfs"+ "github.com/rclone/rclone/vfs/vfscommon""github.com/rclone/rclone/vfs/vfsflags""github.com/spf13/cobra")@@ -61,6 +64,28 @@ func init() {vfsflags.AddFlags(flagSet)proxyflags.AddFlags(flagSet)serve.Command.AddCommand(Command)+ serve.AddRc("s3", func(ctx context.Context, f fs.Fs, in rc.Params) (serve.Handle, error) {+ // Read VFS Opts+ var vfsOpt = vfscommon.Opt // set default opts+ err := configstruct.SetAny(in, &vfsOpt)+ if err != nil {+ return nil, err+ }+ // Read Proxy Opts+ var proxyOpt = proxy.Opt // set default opts+ err = configstruct.SetAny(in, &proxyOpt)+ if err != nil {+ return nil, err+ }+ // Read opts+ var opt = Opt // set default opts+ err = configstruct.SetAny(in, &opt)+ if err != nil {+ return nil, err+ }+ // Create server+ return newServer(ctx, f, &opt, &vfsOpt, &proxyOpt)+ })}//go:embed serve_s3.md@@ -91,18 +116,11 @@ var Command = &cobra.Command{}cmd.Run(false, false, command, func() error {- s, err := newServer(context.Background(), f, &Opt)- if err != nil {- return err- }- router := s.server.Router()- s.Bind(router)- err = s.Serve()+ s, err := newServer(context.Background(), f, &Opt, &vfscommon.Opt, &proxy.Opt)if err != nil {return err}- s.server.Wait()- return nil+ return s.Serve()})return nil},diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.goindex 6bc73085f..15b19d36d 100644--- a/cmd/serve/s3/s3_test.go+++ b/cmd/serve/s3/s3_test.go@@ -24,8 +24,10 @@ import ("github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/object"+ "github.com/rclone/rclone/fs/rc""github.com/rclone/rclone/fstest""github.com/rclone/rclone/lib/random"+ "github.com/rclone/rclone/vfs/vfscommon""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -35,17 +37,16 @@ const ()// Configure and serve the server-func serveS3(f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {+func serveS3(t *testing.T, f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {keyid = random.String(16)keysec = random.String(16)opt := Opt // copy default optionsopt.AuthKey = []string{fmt.Sprintf("%s,%s", keyid, keysec)}opt.HTTP.ListenAddr = []string{endpoint}- w, _ = newServer(context.Background(), f, &opt)- router := w.server.Router()-- w.Bind(router)- _ = w.Serve()+ w, _ = newServer(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt)+ go func() {+ require.NoError(t, w.Serve())+ }()testURL = w.server.URLs()[0]return@@ -55,7 +56,7 @@ func serveS3(f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {// s3 remote against it.func TestS3(t *testing.T) {start := func(f fs.Fs) (configmap.Simple, func()) {- testURL, keyid, keysec, _ := serveS3(f)+ testURL, keyid, keysec, _ := serveS3(t, f)// Config for the backend we'll use to connect to the serverconfig := configmap.Simple{"type": "s3",@@ -118,7 +119,7 @@ func TestEncodingWithMinioClient(t *testing.T) {_, err = f.Put(context.Background(), in, obji)assert.NoError(t, err)- endpoint, keyid, keysec, _ := serveS3(f)+ endpoint, keyid, keysec, _ := serveS3(t, f)testURL, _ := url.Parse(endpoint)minioClient, err := minio.New(testURL.Host, &minio.Options{Creds: credentials.NewStaticV4(keyid, keysec, ""),@@ -181,7 +182,7 @@ func testListBuckets(t *testing.T, cases []TestCase, useProxy bool) {for _, tt := range cases {t.Run(tt.description, func(t *testing.T) {- endpoint, keyid, keysec, s := serveS3(f)+ endpoint, keyid, keysec, s := serveS3(t, f)defer func() {assert.NoError(t, s.server.Shutdown())}()@@ -289,3 +290,10 @@ func TestListBucketsAuthProxy(t *testing.T) {testListBuckets(t, cases, true)}++func TestRc(t *testing.T) {+ servetest.TestRc(t, rc.Params{+ "type": "s3",+ "vfs_cache_mode": "off",+ })+}diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.goindex 05ee994ac..6732eb5e0 100644--- a/cmd/serve/s3/server.go+++ b/cmd/serve/s3/server.go@@ -8,6 +8,7 @@ import ("errors""fmt""math/rand"+ "net""net/http""strings"@@ -43,7 +44,7 @@ type Server struct {}// Make a new S3 Server to serve the remote-func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error) {+func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Options, proxyOpt *proxy.Options) (s *Server, err error) {w := &Server{f: f,ctx: ctx,@@ -84,12 +85,12 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err errorw.handler = w.faker.Server()if proxy.Opt.AuthProxy != "" {- w.proxy = proxy.New(ctx, &proxy.Opt, &vfscommon.Opt)+ w.proxy = proxy.New(ctx, proxyOpt, vfsOpt)// proxy auth middlewarew.handler = proxyAuthMiddleware(w.handler, w)w.handler = authPairMiddleware(w.handler, w)} else {- w._vfs = vfs.New(f, &vfscommon.Opt)+ w._vfs = vfs.New(f, vfsOpt)if len(opt.AuthKey) > 0 {w.faker.AddAuthKeys(authlistResolver(opt.AuthKey))@@ -104,6 +105,9 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err errorreturn nil, fmt.Errorf("failed to init server: %w", err)}+ router := w.server.Router()+ w.Bind(router)+return w, nil}@@ -138,13 +142,24 @@ func (w *Server) Bind(router chi.Router) {router.Handle("/*", w.handler)}-// Serve serves the s3 server+// Serve serves the s3 server until the server is shutdownfunc (w *Server) Serve() error {w.server.Serve()fs.Logf(w.f, "Starting s3 server on %s", w.server.URLs())+ w.server.Wait()return nil}+// Addr returns the first address of the server+func (w *Server) Addr() net.Addr {+ return w.server.Addr()+}++// Shutdown the server+func (w *Server) Shutdown() error {+ return w.server.Shutdown()+}+func authPairMiddleware(next http.Handler, ws *Server) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {accessKey, _ := parseAccessKeyID(r)