Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

s3: fix server side copy failing with --s3-no-head-object - fixes #9629

Anatoly Tarnavsky6df7b8aba1a7
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 791bfc6fc..94b720fae 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3275,6 +3275,16 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
return nil, err
}
+ // With NoHeadObject no metadata was read for the new object, so carry
+ // the size and MD5 over from the source as a server-side copy produces
+ // an object with identical content.
+ if f.opt.NoHeadObject {
+ if dstObject, ok := dstObj.(*Object); ok {
+ dstObject.bytes = srcObj.bytes
+ dstObject.md5 = srcObj.md5
+ }
+ }
+
// Set Object Lock via separate API calls if requested
if f.opt.ObjectLockSetAfterUpload {
if dstObject, ok := dstObj.(*Object); ok {
diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go
index 5ed209b63..797801efd 100644
--- a/backend/s3/s3_internal_test.go
+++ b/backend/s3/s3_internal_test.go
@@ -144,6 +144,33 @@ func (f *Fs) InternalTestNoHead(t *testing.T) {
}
+func (f *Fs) InternalTestNoHeadObjectCopy(t *testing.T) {
+ ctx := context.Background()
+ contents := random.String(1000)
+ item := fstest.NewItem("test-no-head-object-copy-src", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+ src := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+ defer func() {
+ assert.NoError(t, src.Remove(ctx))
+ }()
+ // Set NoHeadObject for this test so the copied object's metadata is not read back
+ f.opt.NoHeadObject = true
+ defer func() {
+ f.opt.NoHeadObject = false
+ }()
+ dst, err := f.Copy(ctx, src, "test-no-head-object-copy-dst")
+ require.NoError(t, err)
+ defer func() {
+ assert.NoError(t, dst.Remove(ctx))
+ }()
+ assert.Equal(t, src.Size(), dst.Size())
+ srcHash, err := src.Hash(ctx, hash.MD5)
+ require.NoError(t, err)
+ dstHash, err := dst.Hash(ctx, hash.MD5)
+ require.NoError(t, err)
+ assert.Equal(t, srcHash, dstHash)
+ assert.NotEqual(t, "", dstHash)
+}
+
func (f *Fs) InternalTestHasChildren(t *testing.T) {
ctx := context.Background()
contents := random.String(100)
@@ -809,6 +836,7 @@ func (f *Fs) InternalTestObjectLock(t *testing.T) {
func (f *Fs) InternalTest(t *testing.T) {
t.Run("Metadata", f.InternalTestMetadata)
t.Run("NoHead", f.InternalTestNoHead)
+ t.Run("NoHeadObjectCopy", f.InternalTestNoHeadObjectCopy)
t.Run("HasChildren", f.InternalTestHasChildren)
t.Run("Versions", f.InternalTestVersions)
t.Run("ObjectLock", f.InternalTestObjectLock)