Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

s3: fix download of compressed files from Cloudflare R2 - fixes #8137

Nick Craig-Wood6079cab0907c
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 3c62dd3c5..8d13f9be9 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -5866,6 +5866,25 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
return resp.Body, err
}
+// middleware to stop the SDK adding `Accept-Encoding: identity`
+func removeDisableGzip() func(*middleware.Stack) error {
+ return func(stack *middleware.Stack) error {
+ _, err := stack.Finalize.Remove("DisableAcceptEncodingGzip")
+ return err
+ }
+}
+
+// middleware to set Accept-Encoding to how we want it
+//
+// This make sure we download compressed files as-is from all platforms
+func (f *Fs) acceptEncoding() (APIOptions []func(*middleware.Stack) error) {
+ APIOptions = append(APIOptions, removeDisableGzip())
+ if f.opt.UseAcceptEncodingGzip.Value {
+ APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
+ }
+ return APIOptions
+}
+
// Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
bucket, bucketPath := o.split()
@@ -5899,11 +5918,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var APIOptions []func(*middleware.Stack) error
- // Override the automatic decompression in the transport to
- // download compressed files as-is
- if o.fs.opt.UseAcceptEncodingGzip.Value {
- APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
- }
+ // Set the SDK to always download compressed files as-is
+ APIOptions = append(APIOptions, o.fs.acceptEncoding()...)
for _, option := range options {
switch option.(type) {
diff --git a/docs/content/s3.md b/docs/content/s3.md
index 58b16b79a..ac62e61ed 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -2503,6 +2503,9 @@ Note that Cloudflare decompresses files uploaded with
does. If this is causing a problem then upload the files with
`--header-upload "Cache-Control: no-transform"`
+A consequence of this is that `Content-Encoding: gzip` will never
+appear in the metadata on Cloudflare.
+
### Dreamhost
Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is