Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: make the ETag of multipart uploads agree with HEAD and support partNumber

Nick Craig-Wood9b5d4567a910
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index cc16e4728..5df593707 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -88,9 +88,11 @@ type tenant struct {
// objectMeta is the metadata stored for an object, with the size and
// modification time the object had when it was stored.
type objectMeta struct {
- meta map[string]string
- size int64
- modTime time.Time
+ meta map[string]string
+ hash []byte // the ETag of the object, if not its hash
+ partSizes []int64 // the sizes of the parts of an object made by a multipart upload
+ size int64
+ modTime time.Time
}
// newMetadataStore makes a store for the metadata of up to maxObjects
@@ -150,18 +152,18 @@ func (t *tenant) context() context.Context {
//
// Nothing is returned if the object has changed since the metadata was
// stored, as it has been replaced by something other than this user.
-func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (meta map[string]string, ok bool) {
- om, ok := t.meta.Get(fp)
+func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (om objectMeta, ok bool) {
+ om, ok = t.meta.Get(fp)
if !ok || om.size != node.Size() {
- return nil, false
+ return objectMeta{}, false
}
window := fs.GetModifyWindow(ctx, node.VFS().Fs())
if window != fs.ModTimeNotSupported {
if dt := node.ModTime().Sub(om.modTime); dt < -window || dt > window {
- return nil, false
+ return objectMeta{}, false
}
}
- return om.meta, true
+ return om, true
}
// storeMeta stores meta as the metadata of the object at fp in _vfs.
@@ -175,6 +177,13 @@ func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (meta ma
// of the request, and caching it would make HeadObject and GetObject
// report that instead of the object's modification time.
func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (err error) {
+ return t.storeObjectMeta(_vfs, fp, objectMeta{meta: meta})
+}
+
+// storeObjectMeta stores om as the metadata of the object at fp in
+// _vfs, as storeMeta does.
+func (t *tenant) storeObjectMeta(_vfs *vfs.VFS, fp string, om objectMeta) (err error) {
+ meta := om.meta
delete(meta, "Last-Modified")
mtime, ok := meta["X-Amz-Meta-Mtime"]
if !ok {
@@ -190,11 +199,8 @@ func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (er
t.removeMeta(fp)
return err
}
- t.meta.Add(fp, objectMeta{
- meta: meta,
- size: node.Size(),
- modTime: node.ModTime(),
- })
+ om.size, om.modTime = node.Size(), node.ModTime()
+ t.meta.Add(fp, om)
return err
}
@@ -217,16 +223,18 @@ func (b *s3Backend) SetObjectTagging(ctx context.Context, bucketName, objectName
return gofakes3.KeyNotFound(objectName)
}
t := b.tenant(ctx)
- meta := map[string]string{}
- if stored, ok := t.getMeta(ctx, fp, node); ok {
- maps.Copy(meta, stored)
+ om, _ := t.getMeta(ctx, fp, node)
+ meta := maps.Clone(om.meta)
+ if meta == nil {
+ meta = map[string]string{}
}
if tagging == "" {
delete(meta, gofakes3.TaggingMetadataKey)
} else {
meta[gofakes3.TaggingMetadataKey] = tagging
}
- return t.storeMeta(_vfs, fp, meta)
+ om.meta = meta
+ return t.storeObjectMeta(_vfs, fp, om)
}
// removeMeta forgets the metadata of the object at fp.
@@ -348,16 +356,22 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin
"Content-Type": mimeType,
}
- if metaMap, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {
- maps.Copy(meta, metaMap)
+ var partSizes []int64
+ if om, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {
+ maps.Copy(meta, om.meta)
+ if om.hash != nil {
+ hash = om.hash
+ }
+ partSizes = om.partSizes
}
return &gofakes3.Object{
- Name: objectName,
- Hash: hash,
- Metadata: meta,
- Size: size,
- Contents: noOpReadCloser{},
+ Name: objectName,
+ Hash: hash,
+ Metadata: meta,
+ Size: size,
+ Contents: noOpReadCloser{},
+ PartSizes: partSizes,
}, nil
}
@@ -437,17 +451,23 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
"Content-Type": mimeType,
}
- if metaMap, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {
- maps.Copy(meta, metaMap)
+ var partSizes []int64
+ if om, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {
+ maps.Copy(meta, om.meta)
+ if om.hash != nil {
+ hash = om.hash
+ }
+ partSizes = om.partSizes
}
return &gofakes3.Object{
- Name: objectName,
- Hash: hash,
- Metadata: meta,
- Size: size,
- Range: rnge,
- Contents: rdr,
+ Name: objectName,
+ Hash: hash,
+ Metadata: meta,
+ Size: size,
+ Range: rnge,
+ Contents: rdr,
+ PartSizes: partSizes,
}, nil
}
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index f915e7fb3..fb7f1caf5 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -35,6 +35,7 @@ import (
"github.com/google/uuid"
"github.com/rclone/gofakes3"
"github.com/rclone/rclone/fs"
+ fshash "github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/pool"
@@ -65,6 +66,8 @@ type multipartUpload struct {
vfs *vfs.VFS // the VFS fh is created on, used for all later operations
tenant *tenant // the user the upload belongs to
+ objectMD5 hash.Hash // MD5 of the data written to the sink, so of the whole object once complete
+
mu sync.Mutex
changed chan struct{} // closed when buffered shrinks, nextPart advances or the upload closes
partMD5s map[int][]byte // raw MD5 sums per part (for the final S3 multipart ETag)
@@ -138,6 +141,7 @@ func newMultipartUpload(bucket, key, fp, streamFp string, meta map[string]string
fp: fp,
streamFp: streamFp,
meta: meta,
+ objectMD5: md5.New(),
partMD5s: map[int][]byte{},
partSizes: map[int]int64{},
nextPart: 1,
@@ -681,9 +685,9 @@ func (up *multipartUpload) stopStreaming(err error) error {
return err
}
-// sink returns the handle the parts are written to, creating it on first
-// use. Call with the sink owned (up.pumping).
-func (up *multipartUpload) sink() (io.WriteCloser, error) {
+// sink returns the writer the parts are written to, creating the handle
+// it writes to on first use. Call with the sink owned (up.pumping).
+func (up *multipartUpload) sink() (io.Writer, error) {
up.mu.Lock()
defer up.mu.Unlock()
if up.closed {
@@ -696,7 +700,20 @@ func (up *multipartUpload) sink() (io.WriteCloser, error) {
}
up.fh = fh
}
- return up.fh, nil
+ return hashingWriter{w: up.fh, hash: up.objectMD5}, nil
+}
+
+// hashingWriter writes to w, adding what is written to hash.
+type hashingWriter struct {
+ w io.Writer
+ hash hash.Hash
+}
+
+// Write writes p to w, adding the bytes written to hash.
+func (h hashingWriter) Write(p []byte) (n int, err error) {
+ n, err = h.w.Write(p)
+ _, _ = h.hash.Write(p[:n])
+ return n, err
}
// copyToSink copies body into sink and hasher, returning the number of
@@ -909,9 +926,35 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
}
b.deleteUpload(uploadID, up)
- _ = up.tenant.storeMeta(up.vfs, up.fp, up.meta)
+ // The ETag is the MD5 of the object when that is the hash used
+ // for ETags, so it is the same as HEAD returns for it, otherwise
+ // the S3 multipart ETag. It is stored, with the part sizes, so
+ // HEAD returns it while the object is unchanged.
+ etag := up.multipartETag(input)
+ var etagHash []byte
+ if b.s.etagHash(up.vfs) == fshash.MD5 {
+ etagHash = up.objectMD5.Sum(nil)
+ etag = fmt.Sprintf("%q", hex.EncodeToString(etagHash))
+ }
+ _ = up.tenant.storeObjectMeta(up.vfs, up.fp, objectMeta{
+ meta: up.meta,
+ hash: etagHash,
+ partSizes: up.listedPartSizes(input),
+ })
+
+ return "", etag, nil
+}
- return "", up.multipartETag(input), nil
+// listedPartSizes returns the sizes of the parts listed in input, in
+// order.
+func (up *multipartUpload) listedPartSizes(input *gofakes3.CompleteMultipartUploadRequest) []int64 {
+ up.mu.Lock()
+ defer up.mu.Unlock()
+ sizes := make([]int64, 0, len(input.Parts))
+ for _, p := range input.Parts {
+ sizes = append(sizes, up.partSizes[p.PartNumber])
+ }
+ return sizes
}
// AbortMultipartUpload tears down an in-progress upload, discarding any data
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index aee900bc2..759aedf03 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -7,6 +7,7 @@ import (
"context"
"crypto/md5"
"encoding/base64"
+ "encoding/hex"
"fmt"
"io"
"math"
@@ -1910,3 +1911,48 @@ func TestDeleteBucketEmptyDirs(t *testing.T) {
_, err = f.List(ctx, bucket)
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
}
+
+// TestMultipartETagAndParts checks the ETag of a completed multipart
+// upload is the MD5 of the object, so is the same as HEAD returns, and
+// that its parts can be read with partNumber.
+func TestMultipartETagAndParts(t *testing.T) {
+ for _, tc := range testRemotes {
+ t.Run(tc.name, func(t *testing.T) {
+ core, _, bucket := newMultipartTestServerBacking(t, tc.backing, false)
+ ctx := context.Background()
+ const object = "multipart.bin"
+ partSizes := []int{120 * 1024, 100 * 1024, 53 * 1024}
+
+ uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})
+ require.NoError(t, err)
+ var want []byte
+ var parts []minio.CompletePart
+ for i, size := range partSizes {
+ data := []byte(random.String(size))
+ want = append(want, data...)
+ p, err := core.PutObjectPart(ctx, bucket, object, uploadID, i+1, bytes.NewReader(data), int64(size), minio.PutObjectPartOptions{})
+ require.NoError(t, err)
+ parts = append(parts, minio.CompletePart{PartNumber: i + 1, ETag: p.ETag})
+ }
+ info, err := core.CompleteMultipartUpload(ctx, bucket, object, uploadID, parts, minio.PutObjectOptions{})
+ require.NoError(t, err)
+ wantMD5 := md5.Sum(want)
+ assert.Equal(t, hex.EncodeToString(wantMD5[:]), info.ETag)
+
+ stat, err := core.StatObject(ctx, bucket, object, minio.StatObjectOptions{})
+ require.NoError(t, err)
+ assert.Equal(t, info.ETag, stat.ETag)
+
+ var start int
+ for i, size := range partSizes {
+ obj, _, _, err := core.GetObject(ctx, bucket, object, minio.GetObjectOptions{PartNumber: i + 1})
+ require.NoError(t, err)
+ data, err := io.ReadAll(obj)
+ require.NoError(t, err)
+ require.NoError(t, obj.Close())
+ assert.Equal(t, want[start:start+size], data, "part %d", i+1)
+ start += size
+ }
+ })
+ }
+}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index d9bb1139a..ac5cd1a88 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -65,6 +65,12 @@ cause problems for S3 clients which rely on the Etag being the MD5.
Use `--etag-hash auto` to use the best hash the backend supports - with
`--auth-proxy` that of each user's backend.
+With `--etag-hash MD5`, the `ETag` of an object made by a multipart
+upload is the MD5 of the whole object, like any other object, rather
+than the `-N` suffixed `ETag` AWS S3 gives these. With another hash
+it is the AWS S3 style multipart `ETag`, but `HeadObject` only
+returns that while the object's metadata is kept (see below).
+
### Quickstart
For a simple set up, to serve `remote:path` over s3, run the server