"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: support metadata setting and mapping on server side Copy
Nick Craig-Woodbda4f25baa18
diff --git a/backend/s3/gen_setfrom.go b/backend/s3/gen_setfrom.goindex b3e0247f2..b70615283 100644--- a/backend/s3/gen_setfrom.go+++ b/backend/s3/gen_setfrom.go@@ -98,4 +98,5 @@ import "github.com/aws/aws-sdk-go/service/s3"genSetFrom(new(s3.HeadObjectOutput), new(s3.GetObjectOutput))genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.PutObjectInput))genSetFrom(new(s3.HeadObjectOutput), new(s3.PutObjectInput))+ genSetFrom(new(s3.CopyObjectInput), new(s3.PutObjectInput))}diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex 9183670c4..63f1f46ba 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -4591,10 +4591,22 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,fs.Debugf(src, "Can't copy - not same remote type")return nil, fs.ErrorCantCopy}+srcBucket, srcPath := srcObj.split()req := s3.CopyObjectInput{MetadataDirective: aws.String(s3.MetadataDirectiveCopy),}++ // Update the metadata if it is in use+ if ci := fs.GetConfig(ctx); ci.Metadata {+ ui, err := srcObj.prepareUpload(ctx, src, fs.MetadataAsOpenOptions(ctx), true)+ if err != nil {+ return nil, fmt.Errorf("failed to prepare upload: %w", err)+ }+ setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)+ req.MetadataDirective = aws.String(s3.MetadataDirectiveReplace)+ }+err = f.copy(ctx, &req, dstBucket, dstPath, srcBucket, srcPath, srcObj)if err != nil {return nil, err@@ -5697,7 +5709,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfs: f,remote: remote,}- ui, err := o.prepareUpload(ctx, src, options)+ ui, err := o.prepareUpload(ctx, src, options, false)if err != nil {return info, nil, fmt.Errorf("failed to prepare upload: %w", err)}@@ -6064,7 +6076,9 @@ type uploadInfo struct {}// Prepare object for being uploaded-func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption) (ui uploadInfo, err error) {+//+// If noHash is true the md5sum will not be calculated+func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, noHash bool) (ui uploadInfo, err error) {bucket, bucketPath := o.split()// Create parent dir/bucket if not saving directory markerif !strings.HasSuffix(o.remote, "/") {@@ -6138,7 +6152,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [var md5sumBase64 stringsize := src.Size()multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)- if !multipart || !o.fs.opt.DisableChecksum {+ if !noHash && (!multipart || !o.fs.opt.DisableChecksum) {ui.md5sumHex, err = src.Hash(ctx, hash.MD5)if err == nil && matchMd5.MatchString(ui.md5sumHex) {hashBytes, err := hex.DecodeString(ui.md5sumHex)@@ -6250,7 +6264,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opif multipart {wantETag, gotETag, versionID, ui, err = o.uploadMultipart(ctx, src, in, options...)} else {- ui, err = o.prepareUpload(ctx, src, options)+ ui, err = o.prepareUpload(ctx, src, options, false)if err != nil {return fmt.Errorf("failed to prepare upload: %w", err)}diff --git a/backend/s3/setfrom.go b/backend/s3/setfrom.goindex 1f7000b24..27eb2aafe 100644--- a/backend/s3/setfrom.go+++ b/backend/s3/setfrom.go@@ -11,6 +11,7 @@ func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b *a.EncodingType = b.EncodingTypea.ExpectedBucketOwner = b.ExpectedBucketOwnera.MaxKeys = b.MaxKeys+ a.OptionalObjectAttributes = b.OptionalObjectAttributesa.Prefix = b.Prefixa.RequestPayer = b.RequestPayer}@@ -25,6 +26,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(a *s3.ListObjectsV2Outputa.MaxKeys = b.MaxKeysa.Name = b.Namea.Prefix = b.Prefix+ a.RequestCharged = b.RequestCharged}// setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b@@ -34,7 +36,9 @@ func setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(a *s3.ListObjectVersa.EncodingType = b.EncodingTypea.ExpectedBucketOwner = b.ExpectedBucketOwnera.MaxKeys = b.MaxKeys+ a.OptionalObjectAttributes = b.OptionalObjectAttributesa.Prefix = b.Prefix+ a.RequestPayer = b.RequestPayer}// setFrom_s3ObjectVersion_s3DeleteMarkerEntry copies matching elements from a to b@@ -55,6 +59,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsVa.MaxKeys = b.MaxKeysa.Name = b.Namea.Prefix = b.Prefix+ a.RequestCharged = b.RequestCharged}// setFrom_s3Object_s3ObjectVersion copies matching elements from a to b@@ -64,6 +69,7 @@ func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) {a.Key = b.Keya.LastModified = b.LastModifieda.Owner = b.Owner+ a.RestoreStatus = b.RestoreStatusa.Size = b.Sizea.StorageClass = b.StorageClass}@@ -237,3 +243,37 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.Pa.StorageClass = b.StorageClassa.WebsiteRedirectLocation = b.WebsiteRedirectLocation}++// setFrom_s3CopyObjectInput_s3PutObjectInput copies matching elements from a to b+func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.PutObjectInput) {+ a.ACL = b.ACL+ a.Bucket = b.Bucket+ a.BucketKeyEnabled = b.BucketKeyEnabled+ a.CacheControl = b.CacheControl+ a.ChecksumAlgorithm = b.ChecksumAlgorithm+ a.ContentDisposition = b.ContentDisposition+ a.ContentEncoding = b.ContentEncoding+ a.ContentLanguage = b.ContentLanguage+ a.ContentType = b.ContentType+ a.ExpectedBucketOwner = b.ExpectedBucketOwner+ a.Expires = b.Expires+ a.GrantFullControl = b.GrantFullControl+ a.GrantRead = b.GrantRead+ a.GrantReadACP = b.GrantReadACP+ a.GrantWriteACP = b.GrantWriteACP+ a.Key = b.Key+ a.Metadata = b.Metadata+ a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus+ a.ObjectLockMode = b.ObjectLockMode+ a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate+ a.RequestPayer = b.RequestPayer+ a.SSECustomerAlgorithm = b.SSECustomerAlgorithm+ a.SSECustomerKey = b.SSECustomerKey+ a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5+ a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext+ a.SSEKMSKeyId = b.SSEKMSKeyId+ a.ServerSideEncryption = b.ServerSideEncryption+ a.StorageClass = b.StorageClass+ a.Tagging = b.Tagging+ a.WebsiteRedirectLocation = b.WebsiteRedirectLocation+}