"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: fix download of compressed files from Cloudflare R2 - fixes #8137
Nick Craig-Wood6079cab0907c
diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex 3c62dd3c5..8d13f9be9 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -5866,6 +5866,25 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, optionsreturn resp.Body, err}+// middleware to stop the SDK adding `Accept-Encoding: identity`+func removeDisableGzip() func(*middleware.Stack) error {+ return func(stack *middleware.Stack) error {+ _, err := stack.Finalize.Remove("DisableAcceptEncodingGzip")+ return err+ }+}++// middleware to set Accept-Encoding to how we want it+//+// This make sure we download compressed files as-is from all platforms+func (f *Fs) acceptEncoding() (APIOptions []func(*middleware.Stack) error) {+ APIOptions = append(APIOptions, removeDisableGzip())+ if f.opt.UseAcceptEncodingGzip.Value {+ APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))+ }+ return APIOptions+}+// Open an object for readfunc (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {bucket, bucketPath := o.split()@@ -5899,11 +5918,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Readvar APIOptions []func(*middleware.Stack) error- // Override the automatic decompression in the transport to- // download compressed files as-is- if o.fs.opt.UseAcceptEncodingGzip.Value {- APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))- }+ // Set the SDK to always download compressed files as-is+ APIOptions = append(APIOptions, o.fs.acceptEncoding()...)for _, option := range options {switch option.(type) {diff --git a/docs/content/s3.md b/docs/content/s3.mdindex 58b16b79a..ac62e61ed 100644--- a/docs/content/s3.md+++ b/docs/content/s3.md@@ -2503,6 +2503,9 @@ Note that Cloudflare decompresses files uploaded withdoes. If this is causing a problem then upload the files with`--header-upload "Cache-Control: no-transform"`+A consequence of this is that `Content-Encoding: gzip` will never+appear in the metadata on Cloudflare.+### DreamhostDreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is