"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: fix --etag-hash auto crashing or using the wrong hash with --auth-proxy
Nick Craig-Wood746eac73efaa
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 3bcb10a01..e1c23e3bc 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -163,7 +163,7 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin// to hashing the VFS cache when the backing object is not available yet.entry := node.DirEntry()size := node.Size()- hash := getFileHashByte(node, b.s.etagHashType)+ hash := getFileHashByte(node, b.s.etagHash(_vfs))mimeType := fs.MimeTypeFromName(objectName)if fobj, ok := entry.(fs.Object); ok {@@ -220,7 +220,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName stringfile := node.(*vfs.File)size := node.Size()- hash := getFileHashByte(node, b.s.etagHashType)+ hash := getFileHashByte(node, b.s.etagHash(_vfs))in, err := file.Open(os.O_RDONLY)if err != nil {diff --git a/cmd/serve/s3/backend_test.go b/cmd/serve/s3/backend_test.goindex 574bbc56c..679506a26 100644--- a/cmd/serve/s3/backend_test.go+++ b/cmd/serve/s3/backend_test.go@@ -2,6 +2,7 @@ package s3import ("context"+ "crypto/md5""io""os""path/filepath"@@ -9,10 +10,14 @@ import ("testing""github.com/rclone/gofakes3"+ _ "github.com/rclone/rclone/backend/crypt"_ "github.com/rclone/rclone/backend/local""github.com/rclone/rclone/cmd/serve/proxy""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/obscure"+ "github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fstest"+ "github.com/rclone/rclone/vfs""github.com/rclone/rclone/vfs/vfscommon""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -180,3 +185,39 @@ func TestBucketDirPath(t *testing.T) {}}}++// TestEtagHashAuto checks that --etag-hash auto uses the best hash of+// the backend of each auth proxy user rather than of the remote the+// server was started with, which may have none or not exist at all.+func TestEtagHashAuto(t *testing.T) {+ fstest.Initialise()+ ctx := context.Background()+ opt := Opt+ opt.EtagHash = "auto"+ opt.HTTP.ListenAddr = []string{endpoint}+ proxyOpt := proxy.Opt+ proxyOpt.AuthProxy = "/path/to/auth/proxy"++ fCrypt, err := fs.NewFs(ctx, ":crypt,remote="+t.TempDir()+",password="+obscure.MustObscure("password")+":")+ require.NoError(t, err)+ require.Equal(t, hash.None, fCrypt.Hashes().GetOne(), "crypt has no hashes")+ for _, f := range []fs.Fs{nil, fCrypt} {+ w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxyOpt)+ require.NoError(t, err)+ t.Cleanup(func() { _ = w.Shutdown() })++ fUser, err := fs.NewFs(ctx, t.TempDir())+ require.NoError(t, err)+ userVFS := vfs.New(ctx, fUser, &vfscommon.Opt)+ t.Cleanup(userVFS.Shutdown)+ ctx := context.WithValue(ctx, ctxKeyID, userVFS)+ require.NoError(t, w.backend.CreateBucket(ctx, "bucket"))+ _, err = w.backend.PutObject(ctx, "bucket", "object", nil, strings.NewReader("data"), 4)+ require.NoError(t, err)++ obj, err := w.backend.HeadObject(ctx, "bucket", "object")+ require.NoError(t, err)+ want := md5.Sum([]byte("data"))+ assert.Equal(t, want[:], obj.Hash)+ }+}diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.goindex 3f7a91572..6b5e79e95 100644--- a/cmd/serve/s3/list.go+++ b/cmd/serve/s3/list.go@@ -8,6 +8,7 @@ import ("github.com/rclone/gofakes3""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/vfs")@@ -27,8 +28,8 @@ var errPageFull = errors.New("listing page full")// so the cost of a page is proportional to the keys it returns rather than to// the size of the subtree below the prefix.type lister struct {- b *s3Backendvfs *vfs.VFS+ hashType hash.Type // hash used for ETagsbucket stringmarker string // keys at or before this one have already been returnedhasMarker bool@@ -49,8 +50,8 @@ func newLister(b *s3Backend, _vfs *vfs.VFS, bucket string, page gofakes3.ListBucmax = 1000}return &lister{- b: b,vfs: _vfs,+ hashType: b.s.etagHash(_vfs),bucket: bucket,marker: page.Marker,hasMarker: page.HasMarker,@@ -114,7 +115,7 @@ func (l *lister) addObject(key string, entry vfs.Node) error {l.response.Add(&gofakes3.Content{Key: key,LastModified: gofakes3.NewContentTime(entry.ModTime()),- ETag: getFileHash(entry, l.b.s.etagHashType),+ ETag: getFileHash(entry, l.hashType),Size: entry.Size(),StorageClass: gofakes3.StorageStandard,})diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 495efb80a..6bfe3aaba 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -53,6 +53,8 @@ part of the hostname (such as mybucket.local)Use `--etag-hash` if you want to change the hash uses for the `ETag`.Note that using anything other than `MD5` (the default) is likely tocause problems for S3 clients which rely on the Etag being the MD5.+Use `--etag-hash auto` to use the best hash the backend supports - with+`--auth-proxy` that of each user's backend.### Quickstartdiff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.goindex eedf65c59..c94e414bc 100644--- a/cmd/serve/s3/server.go+++ b/cmd/serve/s3/server.go@@ -37,7 +37,7 @@ type Server struct {backend *s3Backendhandler http.Handlerctx context.Context // for global config- etagHashType hash.Type+ etagHashType hash.Type // hash for ETags unless opt.EtagHash is "auto"}// Make a new S3 Server to serve the remote@@ -58,9 +58,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt}}()- if w.opt.EtagHash == "auto" {- w.etagHashType = f.Hashes().GetOne()- } else if w.opt.EtagHash != "" {+ if w.opt.EtagHash != "" && w.opt.EtagHash != "auto" {err := w.etagHashType.Set(w.opt.EtagHash)if err != nil {return nil, err@@ -142,6 +140,17 @@ func (w *Server) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {return VFS, nil}+// etagHash returns the hash to use for the ETags of objects in _vfs.+//+// With --etag-hash auto this depends on the backend, which may be+// different for each user of an auth proxy.+func (w *Server) etagHash(_vfs *vfs.VFS) hash.Type {+ if w.opt.EtagHash == "auto" {+ return _vfs.Fs().Hashes().GetOne()+ }+ return w.etagHashType+}+// auth authenticates the request via the auth proxy.//// The proxy maps the access key ID to a VFS and a secret access key