"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: add multi-part-upload support for If-Match and If-None-Match
Sean Turner7cb05a84e960
diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex c9ebca82f..728594864 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -2835,6 +2835,8 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dstSSECustomerKey: req.SSECustomerKey,SSECustomerKeyMD5: req.SSECustomerKeyMD5,UploadId: uid,+ IfMatch: copyReq.IfMatch,+ IfNoneMatch: copyReq.IfNoneMatch,})return f.shouldRetry(ctx, err)})@@ -2869,13 +2871,20 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,MetadataDirective: types.MetadataDirectiveCopy,}- // Update the metadata if it is in use- if ci := fs.GetConfig(ctx); ci.Metadata {- ui, err := srcObj.prepareUpload(ctx, src, fs.MetadataAsOpenOptions(ctx), true)- if err != nil {- return nil, fmt.Errorf("failed to prepare upload: %w", err)- }- setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)+ // Build upload options including headers and metadata+ ci := fs.GetConfig(ctx)+ uploadOptions := fs.MetadataAsOpenOptions(ctx)+ for _, option := range ci.UploadHeaders {+ uploadOptions = append(uploadOptions, option)+ }++ ui, err := srcObj.prepareUpload(ctx, src, uploadOptions, true)+ if err != nil {+ return nil, fmt.Errorf("failed to prepare upload: %w", err)+ }++ setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)+ if ci.Metadata {req.MetadataDirective = types.MetadataDirectiveReplace}@@ -4284,6 +4293,8 @@ func (w *s3ChunkWriter) Close(ctx context.Context) (err error) {SSECustomerKey: w.multiPartUploadInput.SSECustomerKey,SSECustomerKeyMD5: w.multiPartUploadInput.SSECustomerKeyMD5,UploadId: w.uploadID,+ IfMatch: w.ui.req.IfMatch,+ IfNoneMatch: w.ui.req.IfNoneMatch,})return w.f.shouldRetry(ctx, err)})diff --git a/backend/s3/setfrom.go b/backend/s3/setfrom.goindex 439e8b6b0..a0c94a4c4 100644--- a/backend/s3/setfrom.go+++ b/backend/s3/setfrom.go@@ -70,6 +70,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV// setFrom_typesObject_typesObjectVersion copies matching elements from a to bfunc setFrom_typesObject_typesObjectVersion(a *types.Object, b *types.ObjectVersion) {a.ChecksumAlgorithm = b.ChecksumAlgorithm+ a.ChecksumType = b.ChecksumTypea.ETag = b.ETaga.Key = b.Keya.LastModified = b.LastModified@@ -82,6 +83,7 @@ func setFrom_typesObject_typesObjectVersion(a *types.Object, b *types.ObjectVersfunc setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipartUploadInput, b *s3.HeadObjectOutput) {a.BucketKeyEnabled = b.BucketKeyEnableda.CacheControl = b.CacheControl+ a.ChecksumType = b.ChecksumTypea.ContentDisposition = b.ContentDispositiona.ContentEncoding = b.ContentEncodinga.ContentLanguage = b.ContentLanguage@@ -160,12 +162,15 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.a.CacheControl = b.CacheControla.ChecksumCRC32 = b.ChecksumCRC32a.ChecksumCRC32C = b.ChecksumCRC32C+ a.ChecksumCRC64NVME = b.ChecksumCRC64NVMEa.ChecksumSHA1 = b.ChecksumSHA1a.ChecksumSHA256 = b.ChecksumSHA256+ a.ChecksumType = b.ChecksumTypea.ContentDisposition = b.ContentDispositiona.ContentEncoding = b.ContentEncodinga.ContentLanguage = b.ContentLanguagea.ContentLength = b.ContentLength+ a.ContentRange = b.ContentRangea.ContentType = b.ContentTypea.DeleteMarker = b.DeleteMarkera.ETag = b.ETag@@ -187,6 +192,7 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.a.SSEKMSKeyId = b.SSEKMSKeyIda.ServerSideEncryption = b.ServerSideEncryptiona.StorageClass = b.StorageClass+ a.TagCount = b.TagCounta.VersionId = b.VersionIda.WebsiteRedirectLocation = b.WebsiteRedirectLocationa.ResultMetadata = b.ResultMetadata@@ -232,6 +238,7 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.Pa.CacheControl = b.CacheControla.ChecksumCRC32 = b.ChecksumCRC32a.ChecksumCRC32C = b.ChecksumCRC32C+ a.ChecksumCRC64NVME = b.ChecksumCRC64NVMEa.ChecksumSHA1 = b.ChecksumSHA1a.ChecksumSHA256 = b.ChecksumSHA256a.ContentDisposition = b.ContentDisposition@@ -270,6 +277,8 @@ func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.Puta.GrantRead = b.GrantReada.GrantReadACP = b.GrantReadACPa.GrantWriteACP = b.GrantWriteACP+ a.IfMatch = b.IfMatch+ a.IfNoneMatch = b.IfNoneMatcha.Metadata = b.Metadataa.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatusa.ObjectLockMode = b.ObjectLockMode