Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

s3: support metadata setting and mapping on server side Copy

Nick Craig-Woodbda4f25baa18
diff --git a/backend/s3/gen_setfrom.go b/backend/s3/gen_setfrom.go
index b3e0247f2..b70615283 100644
--- a/backend/s3/gen_setfrom.go
+++ b/backend/s3/gen_setfrom.go
@@ -98,4 +98,5 @@ import "github.com/aws/aws-sdk-go/service/s3"
genSetFrom(new(s3.HeadObjectOutput), new(s3.GetObjectOutput))
genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.PutObjectInput))
genSetFrom(new(s3.HeadObjectOutput), new(s3.PutObjectInput))
+ genSetFrom(new(s3.CopyObjectInput), new(s3.PutObjectInput))
}
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 9183670c4..63f1f46ba 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -4591,10 +4591,22 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
+
srcBucket, srcPath := srcObj.split()
req := s3.CopyObjectInput{
MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
}
+
+ // Update the metadata if it is in use
+ if ci := fs.GetConfig(ctx); ci.Metadata {
+ ui, err := srcObj.prepareUpload(ctx, src, fs.MetadataAsOpenOptions(ctx), true)
+ if err != nil {
+ return nil, fmt.Errorf("failed to prepare upload: %w", err)
+ }
+ setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)
+ req.MetadataDirective = aws.String(s3.MetadataDirectiveReplace)
+ }
+
err = f.copy(ctx, &req, dstBucket, dstPath, srcBucket, srcPath, srcObj)
if err != nil {
return nil, err
@@ -5697,7 +5709,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
fs: f,
remote: remote,
}
- ui, err := o.prepareUpload(ctx, src, options)
+ ui, err := o.prepareUpload(ctx, src, options, false)
if err != nil {
return info, nil, fmt.Errorf("failed to prepare upload: %w", err)
}
@@ -6064,7 +6076,9 @@ type uploadInfo struct {
}
// Prepare object for being uploaded
-func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption) (ui uploadInfo, err error) {
+//
+// If noHash is true the md5sum will not be calculated
+func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, noHash bool) (ui uploadInfo, err error) {
bucket, bucketPath := o.split()
// Create parent dir/bucket if not saving directory marker
if !strings.HasSuffix(o.remote, "/") {
@@ -6138,7 +6152,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
var md5sumBase64 string
size := src.Size()
multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
- if !multipart || !o.fs.opt.DisableChecksum {
+ if !noHash && (!multipart || !o.fs.opt.DisableChecksum) {
ui.md5sumHex, err = src.Hash(ctx, hash.MD5)
if err == nil && matchMd5.MatchString(ui.md5sumHex) {
hashBytes, err := hex.DecodeString(ui.md5sumHex)
@@ -6250,7 +6264,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if multipart {
wantETag, gotETag, versionID, ui, err = o.uploadMultipart(ctx, src, in, options...)
} else {
- ui, err = o.prepareUpload(ctx, src, options)
+ ui, err = o.prepareUpload(ctx, src, options, false)
if err != nil {
return fmt.Errorf("failed to prepare upload: %w", err)
}
diff --git a/backend/s3/setfrom.go b/backend/s3/setfrom.go
index 1f7000b24..27eb2aafe 100644
--- a/backend/s3/setfrom.go
+++ b/backend/s3/setfrom.go
@@ -11,6 +11,7 @@ func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b *
a.EncodingType = b.EncodingType
a.ExpectedBucketOwner = b.ExpectedBucketOwner
a.MaxKeys = b.MaxKeys
+ a.OptionalObjectAttributes = b.OptionalObjectAttributes
a.Prefix = b.Prefix
a.RequestPayer = b.RequestPayer
}
@@ -25,6 +26,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(a *s3.ListObjectsV2Output
a.MaxKeys = b.MaxKeys
a.Name = b.Name
a.Prefix = b.Prefix
+ a.RequestCharged = b.RequestCharged
}
// setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b
@@ -34,7 +36,9 @@ func setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(a *s3.ListObjectVers
a.EncodingType = b.EncodingType
a.ExpectedBucketOwner = b.ExpectedBucketOwner
a.MaxKeys = b.MaxKeys
+ a.OptionalObjectAttributes = b.OptionalObjectAttributes
a.Prefix = b.Prefix
+ a.RequestPayer = b.RequestPayer
}
// setFrom_s3ObjectVersion_s3DeleteMarkerEntry copies matching elements from a to b
@@ -55,6 +59,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV
a.MaxKeys = b.MaxKeys
a.Name = b.Name
a.Prefix = b.Prefix
+ a.RequestCharged = b.RequestCharged
}
// setFrom_s3Object_s3ObjectVersion copies matching elements from a to b
@@ -64,6 +69,7 @@ func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) {
a.Key = b.Key
a.LastModified = b.LastModified
a.Owner = b.Owner
+ a.RestoreStatus = b.RestoreStatus
a.Size = b.Size
a.StorageClass = b.StorageClass
}
@@ -237,3 +243,37 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.P
a.StorageClass = b.StorageClass
a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
}
+
+// setFrom_s3CopyObjectInput_s3PutObjectInput copies matching elements from a to b
+func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.PutObjectInput) {
+ a.ACL = b.ACL
+ a.Bucket = b.Bucket
+ a.BucketKeyEnabled = b.BucketKeyEnabled
+ a.CacheControl = b.CacheControl
+ a.ChecksumAlgorithm = b.ChecksumAlgorithm
+ a.ContentDisposition = b.ContentDisposition
+ a.ContentEncoding = b.ContentEncoding
+ a.ContentLanguage = b.ContentLanguage
+ a.ContentType = b.ContentType
+ a.ExpectedBucketOwner = b.ExpectedBucketOwner
+ a.Expires = b.Expires
+ a.GrantFullControl = b.GrantFullControl
+ a.GrantRead = b.GrantRead
+ a.GrantReadACP = b.GrantReadACP
+ a.GrantWriteACP = b.GrantWriteACP
+ a.Key = b.Key
+ a.Metadata = b.Metadata
+ a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
+ a.ObjectLockMode = b.ObjectLockMode
+ a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate
+ a.RequestPayer = b.RequestPayer
+ a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
+ a.SSECustomerKey = b.SSECustomerKey
+ a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5
+ a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext
+ a.SSEKMSKeyId = b.SSEKMSKeyId
+ a.ServerSideEncryption = b.ServerSideEncryption
+ a.StorageClass = b.StorageClass
+ a.Tagging = b.Tagging
+ a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
+}