"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: fix Content-Type: aws-chunked causing upload errors with --metadata
Nick Craig-Woodcb1507fa96a3
diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex a1379da3d..de06383af 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -6086,7 +6086,7 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {o.storageClass = stringClone(string(resp.StorageClass))o.cacheControl = stringClonePointer(resp.CacheControl)o.contentDisposition = stringClonePointer(resp.ContentDisposition)- o.contentEncoding = stringClonePointer(resp.ContentEncoding)+ o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding))o.contentLanguage = stringClonePointer(resp.ContentLanguage)// If decompressing then size and md5sum are unknown@@ -6154,6 +6154,36 @@ func (o *Object) Storable() bool {return true}+// removeAWSChunked removes the "aws-chunked" content-coding from a+// Content-Encoding field value (RFC 9110). Comparison is case-insensitive.+// Returns nil if encoding is empty after removal.+func removeAWSChunked(pv *string) *string {+ if pv == nil {+ return nil+ }+ v := *pv+ if v == "" {+ return nil+ }+ if !strings.Contains(strings.ToLower(v), "aws-chunked") {+ return pv+ }+ parts := strings.Split(v, ",")+ out := make([]string, 0, len(parts))+ for _, p := range parts {+ tok := strings.TrimSpace(p)+ if tok == "" || strings.EqualFold(tok, "aws-chunked") {+ continue+ }+ out = append(out, tok)+ }+ if len(out) == 0 {+ return nil+ }+ v = strings.Join(out, ",")+ return &v+}+func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options ...fs.OpenOption) (in io.ReadCloser, err error) {url := o.fs.opt.DownloadURL + bucketPathvar resp *http.Response@@ -6322,7 +6352,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Reado.setMetaData(&head)// Decompress body if necessary- if deref(resp.ContentEncoding) == "gzip" {+ if deref(removeAWSChunked(resp.ContentEncoding)) == "gzip" {if o.fs.opt.Decompress || (resp.ContentLength == nil && o.fs.opt.MightGzip.Value) {return readers.NewGzipReader(resp.Body)}@@ -6746,7 +6776,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [case "content-disposition":ui.req.ContentDisposition = pvcase "content-encoding":- ui.req.ContentEncoding = pv+ ui.req.ContentEncoding = removeAWSChunked(pv)case "content-language":ui.req.ContentLanguage = pvcase "content-type":@@ -6843,7 +6873,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [case "content-disposition":ui.req.ContentDisposition = aws.String(value)case "content-encoding":- ui.req.ContentEncoding = aws.String(value)+ ui.req.ContentEncoding = removeAWSChunked(aws.String(value))case "content-language":ui.req.ContentLanguage = aws.String(value)case "content-type":diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.goindex 86e23e436..cdca649ae 100644--- a/backend/s3/s3_internal_test.go+++ b/backend/s3/s3_internal_test.go@@ -248,6 +248,47 @@ func TestMergeDeleteMarkers(t *testing.T) {}}+func TestRemoveAWSChunked(t *testing.T) {+ ps := func(s string) *string {+ return &s+ }+ tests := []struct {+ name string+ in *string+ want *string+ }{+ {"nil", nil, nil},+ {"empty", ps(""), nil},+ {"only aws", ps("aws-chunked"), nil},+ {"leading aws", ps("aws-chunked, gzip"), ps("gzip")},+ {"trailing aws", ps("gzip, aws-chunked"), ps("gzip")},+ {"middle aws", ps("gzip, aws-chunked, br"), ps("gzip,br")},+ {"case insensitive", ps("GZip, AwS-ChUnKeD, Br"), ps("GZip,Br")},+ {"duplicates", ps("aws-chunked , aws-chunked"), nil},+ {"no aws normalize spaces", ps(" gzip , br "), ps(" gzip , br ")},+ {"surrounding spaces", ps(" aws-chunked "), nil},+ {"no change", ps("gzip, br"), ps("gzip, br")},+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ got := removeAWSChunked(tc.in)+ check := func(want, got *string) {+ t.Helper()+ if tc.want == nil {+ assert.Nil(t, got)+ } else {+ require.NotNil(t, got)+ assert.Equal(t, *tc.want, *got)+ }+ }+ check(tc.want, got)+ // Idempotent+ got2 := removeAWSChunked(got)+ check(got, got2)+ })+ }+}+func (f *Fs) InternalTestVersions(t *testing.T) {ctx := context.Background()