Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

s3: add multi-part-upload support for If-Match and If-None-Match

Sean Turner7cb05a84e960
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index c9ebca82f..728594864 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -2835,6 +2835,8 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
SSECustomerKey: req.SSECustomerKey,
SSECustomerKeyMD5: req.SSECustomerKeyMD5,
UploadId: uid,
+ IfMatch: copyReq.IfMatch,
+ IfNoneMatch: copyReq.IfNoneMatch,
})
return f.shouldRetry(ctx, err)
})
@@ -2869,13 +2871,20 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
MetadataDirective: types.MetadataDirectiveCopy,
}
- // Update the metadata if it is in use
- if ci := fs.GetConfig(ctx); ci.Metadata {
- ui, err := srcObj.prepareUpload(ctx, src, fs.MetadataAsOpenOptions(ctx), true)
- if err != nil {
- return nil, fmt.Errorf("failed to prepare upload: %w", err)
- }
- setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)
+ // Build upload options including headers and metadata
+ ci := fs.GetConfig(ctx)
+ uploadOptions := fs.MetadataAsOpenOptions(ctx)
+ for _, option := range ci.UploadHeaders {
+ uploadOptions = append(uploadOptions, option)
+ }
+
+ ui, err := srcObj.prepareUpload(ctx, src, uploadOptions, true)
+ if err != nil {
+ return nil, fmt.Errorf("failed to prepare upload: %w", err)
+ }
+
+ setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)
+ if ci.Metadata {
req.MetadataDirective = types.MetadataDirectiveReplace
}
@@ -4284,6 +4293,8 @@ func (w *s3ChunkWriter) Close(ctx context.Context) (err error) {
SSECustomerKey: w.multiPartUploadInput.SSECustomerKey,
SSECustomerKeyMD5: w.multiPartUploadInput.SSECustomerKeyMD5,
UploadId: w.uploadID,
+ IfMatch: w.ui.req.IfMatch,
+ IfNoneMatch: w.ui.req.IfNoneMatch,
})
return w.f.shouldRetry(ctx, err)
})
diff --git a/backend/s3/setfrom.go b/backend/s3/setfrom.go
index 439e8b6b0..a0c94a4c4 100644
--- a/backend/s3/setfrom.go
+++ b/backend/s3/setfrom.go
@@ -70,6 +70,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV
// setFrom_typesObject_typesObjectVersion copies matching elements from a to b
func setFrom_typesObject_typesObjectVersion(a *types.Object, b *types.ObjectVersion) {
a.ChecksumAlgorithm = b.ChecksumAlgorithm
+ a.ChecksumType = b.ChecksumType
a.ETag = b.ETag
a.Key = b.Key
a.LastModified = b.LastModified
@@ -82,6 +83,7 @@ func setFrom_typesObject_typesObjectVersion(a *types.Object, b *types.ObjectVers
func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipartUploadInput, b *s3.HeadObjectOutput) {
a.BucketKeyEnabled = b.BucketKeyEnabled
a.CacheControl = b.CacheControl
+ a.ChecksumType = b.ChecksumType
a.ContentDisposition = b.ContentDisposition
a.ContentEncoding = b.ContentEncoding
a.ContentLanguage = b.ContentLanguage
@@ -160,12 +162,15 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.
a.CacheControl = b.CacheControl
a.ChecksumCRC32 = b.ChecksumCRC32
a.ChecksumCRC32C = b.ChecksumCRC32C
+ a.ChecksumCRC64NVME = b.ChecksumCRC64NVME
a.ChecksumSHA1 = b.ChecksumSHA1
a.ChecksumSHA256 = b.ChecksumSHA256
+ a.ChecksumType = b.ChecksumType
a.ContentDisposition = b.ContentDisposition
a.ContentEncoding = b.ContentEncoding
a.ContentLanguage = b.ContentLanguage
a.ContentLength = b.ContentLength
+ a.ContentRange = b.ContentRange
a.ContentType = b.ContentType
a.DeleteMarker = b.DeleteMarker
a.ETag = b.ETag
@@ -187,6 +192,7 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.
a.SSEKMSKeyId = b.SSEKMSKeyId
a.ServerSideEncryption = b.ServerSideEncryption
a.StorageClass = b.StorageClass
+ a.TagCount = b.TagCount
a.VersionId = b.VersionId
a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
a.ResultMetadata = b.ResultMetadata
@@ -232,6 +238,7 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.P
a.CacheControl = b.CacheControl
a.ChecksumCRC32 = b.ChecksumCRC32
a.ChecksumCRC32C = b.ChecksumCRC32C
+ a.ChecksumCRC64NVME = b.ChecksumCRC64NVME
a.ChecksumSHA1 = b.ChecksumSHA1
a.ChecksumSHA256 = b.ChecksumSHA256
a.ContentDisposition = b.ContentDisposition
@@ -270,6 +277,8 @@ func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.Put
a.GrantRead = b.GrantRead
a.GrantReadACP = b.GrantReadACP
a.GrantWriteACP = b.GrantWriteACP
+ a.IfMatch = b.IfMatch
+ a.IfNoneMatch = b.IfNoneMatch
a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode