"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: make the ETag of multipart uploads agree with HEAD and support partNumber
Nick Craig-Wood9b5d4567a910
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex cc16e4728..5df593707 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -88,9 +88,11 @@ type tenant struct {// objectMeta is the metadata stored for an object, with the size and// modification time the object had when it was stored.type objectMeta struct {- meta map[string]string- size int64- modTime time.Time+ meta map[string]string+ hash []byte // the ETag of the object, if not its hash+ partSizes []int64 // the sizes of the parts of an object made by a multipart upload+ size int64+ modTime time.Time}// newMetadataStore makes a store for the metadata of up to maxObjects@@ -150,18 +152,18 @@ func (t *tenant) context() context.Context {//// Nothing is returned if the object has changed since the metadata was// stored, as it has been replaced by something other than this user.-func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (meta map[string]string, ok bool) {- om, ok := t.meta.Get(fp)+func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (om objectMeta, ok bool) {+ om, ok = t.meta.Get(fp)if !ok || om.size != node.Size() {- return nil, false+ return objectMeta{}, false}window := fs.GetModifyWindow(ctx, node.VFS().Fs())if window != fs.ModTimeNotSupported {if dt := node.ModTime().Sub(om.modTime); dt < -window || dt > window {- return nil, false+ return objectMeta{}, false}}- return om.meta, true+ return om, true}// storeMeta stores meta as the metadata of the object at fp in _vfs.@@ -175,6 +177,13 @@ func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (meta ma// of the request, and caching it would make HeadObject and GetObject// report that instead of the object's modification time.func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (err error) {+ return t.storeObjectMeta(_vfs, fp, objectMeta{meta: meta})+}++// storeObjectMeta stores om as the metadata of the object at fp in+// _vfs, as storeMeta does.+func (t *tenant) storeObjectMeta(_vfs *vfs.VFS, fp string, om objectMeta) (err error) {+ meta := om.metadelete(meta, "Last-Modified")mtime, ok := meta["X-Amz-Meta-Mtime"]if !ok {@@ -190,11 +199,8 @@ func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (ert.removeMeta(fp)return err}- t.meta.Add(fp, objectMeta{- meta: meta,- size: node.Size(),- modTime: node.ModTime(),- })+ om.size, om.modTime = node.Size(), node.ModTime()+ t.meta.Add(fp, om)return err}@@ -217,16 +223,18 @@ func (b *s3Backend) SetObjectTagging(ctx context.Context, bucketName, objectNamereturn gofakes3.KeyNotFound(objectName)}t := b.tenant(ctx)- meta := map[string]string{}- if stored, ok := t.getMeta(ctx, fp, node); ok {- maps.Copy(meta, stored)+ om, _ := t.getMeta(ctx, fp, node)+ meta := maps.Clone(om.meta)+ if meta == nil {+ meta = map[string]string{}}if tagging == "" {delete(meta, gofakes3.TaggingMetadataKey)} else {meta[gofakes3.TaggingMetadataKey] = tagging}- return t.storeMeta(_vfs, fp, meta)+ om.meta = meta+ return t.storeObjectMeta(_vfs, fp, om)}// removeMeta forgets the metadata of the object at fp.@@ -348,16 +356,22 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin"Content-Type": mimeType,}- if metaMap, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {- maps.Copy(meta, metaMap)+ var partSizes []int64+ if om, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {+ maps.Copy(meta, om.meta)+ if om.hash != nil {+ hash = om.hash+ }+ partSizes = om.partSizes}return &gofakes3.Object{- Name: objectName,- Hash: hash,- Metadata: meta,- Size: size,- Contents: noOpReadCloser{},+ Name: objectName,+ Hash: hash,+ Metadata: meta,+ Size: size,+ Contents: noOpReadCloser{},+ PartSizes: partSizes,}, nil}@@ -437,17 +451,23 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string"Content-Type": mimeType,}- if metaMap, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {- maps.Copy(meta, metaMap)+ var partSizes []int64+ if om, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {+ maps.Copy(meta, om.meta)+ if om.hash != nil {+ hash = om.hash+ }+ partSizes = om.partSizes}return &gofakes3.Object{- Name: objectName,- Hash: hash,- Metadata: meta,- Size: size,- Range: rnge,- Contents: rdr,+ Name: objectName,+ Hash: hash,+ Metadata: meta,+ Size: size,+ Range: rnge,+ Contents: rdr,+ PartSizes: partSizes,}, nil}diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex f915e7fb3..fb7f1caf5 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -35,6 +35,7 @@ import ("github.com/google/uuid""github.com/rclone/gofakes3""github.com/rclone/rclone/fs"+ fshash "github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/pool"@@ -65,6 +66,8 @@ type multipartUpload struct {vfs *vfs.VFS // the VFS fh is created on, used for all later operationstenant *tenant // the user the upload belongs to+ objectMD5 hash.Hash // MD5 of the data written to the sink, so of the whole object once complete+mu sync.Mutexchanged chan struct{} // closed when buffered shrinks, nextPart advances or the upload closespartMD5s map[int][]byte // raw MD5 sums per part (for the final S3 multipart ETag)@@ -138,6 +141,7 @@ func newMultipartUpload(bucket, key, fp, streamFp string, meta map[string]stringfp: fp,streamFp: streamFp,meta: meta,+ objectMD5: md5.New(),partMD5s: map[int][]byte{},partSizes: map[int]int64{},nextPart: 1,@@ -681,9 +685,9 @@ func (up *multipartUpload) stopStreaming(err error) error {return err}-// sink returns the handle the parts are written to, creating it on first-// use. Call with the sink owned (up.pumping).-func (up *multipartUpload) sink() (io.WriteCloser, error) {+// sink returns the writer the parts are written to, creating the handle+// it writes to on first use. Call with the sink owned (up.pumping).+func (up *multipartUpload) sink() (io.Writer, error) {up.mu.Lock()defer up.mu.Unlock()if up.closed {@@ -696,7 +700,20 @@ func (up *multipartUpload) sink() (io.WriteCloser, error) {}up.fh = fh}- return up.fh, nil+ return hashingWriter{w: up.fh, hash: up.objectMD5}, nil+}++// hashingWriter writes to w, adding what is written to hash.+type hashingWriter struct {+ w io.Writer+ hash hash.Hash+}++// Write writes p to w, adding the bytes written to hash.+func (h hashingWriter) Write(p []byte) (n int, err error) {+ n, err = h.w.Write(p)+ _, _ = h.hash.Write(p[:n])+ return n, err}// copyToSink copies body into sink and hasher, returning the number of@@ -909,9 +926,35 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj}b.deleteUpload(uploadID, up)- _ = up.tenant.storeMeta(up.vfs, up.fp, up.meta)+ // The ETag is the MD5 of the object when that is the hash used+ // for ETags, so it is the same as HEAD returns for it, otherwise+ // the S3 multipart ETag. It is stored, with the part sizes, so+ // HEAD returns it while the object is unchanged.+ etag := up.multipartETag(input)+ var etagHash []byte+ if b.s.etagHash(up.vfs) == fshash.MD5 {+ etagHash = up.objectMD5.Sum(nil)+ etag = fmt.Sprintf("%q", hex.EncodeToString(etagHash))+ }+ _ = up.tenant.storeObjectMeta(up.vfs, up.fp, objectMeta{+ meta: up.meta,+ hash: etagHash,+ partSizes: up.listedPartSizes(input),+ })++ return "", etag, nil+}- return "", up.multipartETag(input), nil+// listedPartSizes returns the sizes of the parts listed in input, in+// order.+func (up *multipartUpload) listedPartSizes(input *gofakes3.CompleteMultipartUploadRequest) []int64 {+ up.mu.Lock()+ defer up.mu.Unlock()+ sizes := make([]int64, 0, len(input.Parts))+ for _, p := range input.Parts {+ sizes = append(sizes, up.partSizes[p.PartNumber])+ }+ return sizes}// AbortMultipartUpload tears down an in-progress upload, discarding any datadiff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex aee900bc2..759aedf03 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -7,6 +7,7 @@ import ("context""crypto/md5""encoding/base64"+ "encoding/hex""fmt""io""math"@@ -1910,3 +1911,48 @@ func TestDeleteBucketEmptyDirs(t *testing.T) {_, err = f.List(ctx, bucket)assert.ErrorIs(t, err, fs.ErrorDirNotFound)}++// TestMultipartETagAndParts checks the ETag of a completed multipart+// upload is the MD5 of the object, so is the same as HEAD returns, and+// that its parts can be read with partNumber.+func TestMultipartETagAndParts(t *testing.T) {+ for _, tc := range testRemotes {+ t.Run(tc.name, func(t *testing.T) {+ core, _, bucket := newMultipartTestServerBacking(t, tc.backing, false)+ ctx := context.Background()+ const object = "multipart.bin"+ partSizes := []int{120 * 1024, 100 * 1024, 53 * 1024}++ uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})+ require.NoError(t, err)+ var want []byte+ var parts []minio.CompletePart+ for i, size := range partSizes {+ data := []byte(random.String(size))+ want = append(want, data...)+ p, err := core.PutObjectPart(ctx, bucket, object, uploadID, i+1, bytes.NewReader(data), int64(size), minio.PutObjectPartOptions{})+ require.NoError(t, err)+ parts = append(parts, minio.CompletePart{PartNumber: i + 1, ETag: p.ETag})+ }+ info, err := core.CompleteMultipartUpload(ctx, bucket, object, uploadID, parts, minio.PutObjectOptions{})+ require.NoError(t, err)+ wantMD5 := md5.Sum(want)+ assert.Equal(t, hex.EncodeToString(wantMD5[:]), info.ETag)++ stat, err := core.StatObject(ctx, bucket, object, minio.StatObjectOptions{})+ require.NoError(t, err)+ assert.Equal(t, info.ETag, stat.ETag)++ var start int+ for i, size := range partSizes {+ obj, _, _, err := core.GetObject(ctx, bucket, object, minio.GetObjectOptions{PartNumber: i + 1})+ require.NoError(t, err)+ data, err := io.ReadAll(obj)+ require.NoError(t, err)+ require.NoError(t, obj.Close())+ assert.Equal(t, want[start:start+size], data, "part %d", i+1)+ start += size+ }+ })+ }+}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex d9bb1139a..ac5cd1a88 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -65,6 +65,12 @@ cause problems for S3 clients which rely on the Etag being the MD5.Use `--etag-hash auto` to use the best hash the backend supports - with`--auth-proxy` that of each user's backend.+With `--etag-hash MD5`, the `ETag` of an object made by a multipart+upload is the MD5 of the whole object, like any other object, rather+than the `-N` suffixed `ETag` AWS S3 gives these. With another hash+it is the AWS S3 style multipart `ETag`, but `HeadObject` only+returns that while the object's metadata is kept (see below).+### QuickstartFor a simple set up, to serve `remote:path` over s3, run the server