Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

s3: fix Content-Type: aws-chunked causing upload errors with --metadata

Nick Craig-Woodcb1507fa96a3
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index a1379da3d..de06383af 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -6086,7 +6086,7 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {
o.storageClass = stringClone(string(resp.StorageClass))
o.cacheControl = stringClonePointer(resp.CacheControl)
o.contentDisposition = stringClonePointer(resp.ContentDisposition)
- o.contentEncoding = stringClonePointer(resp.ContentEncoding)
+ o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding))
o.contentLanguage = stringClonePointer(resp.ContentLanguage)
// If decompressing then size and md5sum are unknown
@@ -6154,6 +6154,36 @@ func (o *Object) Storable() bool {
return true
}
+// removeAWSChunked removes the "aws-chunked" content-coding from a
+// Content-Encoding field value (RFC 9110). Comparison is case-insensitive.
+// Returns nil if encoding is empty after removal.
+func removeAWSChunked(pv *string) *string {
+ if pv == nil {
+ return nil
+ }
+ v := *pv
+ if v == "" {
+ return nil
+ }
+ if !strings.Contains(strings.ToLower(v), "aws-chunked") {
+ return pv
+ }
+ parts := strings.Split(v, ",")
+ out := make([]string, 0, len(parts))
+ for _, p := range parts {
+ tok := strings.TrimSpace(p)
+ if tok == "" || strings.EqualFold(tok, "aws-chunked") {
+ continue
+ }
+ out = append(out, tok)
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ v = strings.Join(out, ",")
+ return &v
+}
+
func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
url := o.fs.opt.DownloadURL + bucketPath
var resp *http.Response
@@ -6322,7 +6352,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
o.setMetaData(&head)
// Decompress body if necessary
- if deref(resp.ContentEncoding) == "gzip" {
+ if deref(removeAWSChunked(resp.ContentEncoding)) == "gzip" {
if o.fs.opt.Decompress || (resp.ContentLength == nil && o.fs.opt.MightGzip.Value) {
return readers.NewGzipReader(resp.Body)
}
@@ -6746,7 +6776,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
case "content-disposition":
ui.req.ContentDisposition = pv
case "content-encoding":
- ui.req.ContentEncoding = pv
+ ui.req.ContentEncoding = removeAWSChunked(pv)
case "content-language":
ui.req.ContentLanguage = pv
case "content-type":
@@ -6843,7 +6873,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
case "content-disposition":
ui.req.ContentDisposition = aws.String(value)
case "content-encoding":
- ui.req.ContentEncoding = aws.String(value)
+ ui.req.ContentEncoding = removeAWSChunked(aws.String(value))
case "content-language":
ui.req.ContentLanguage = aws.String(value)
case "content-type":
diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go
index 86e23e436..cdca649ae 100644
--- a/backend/s3/s3_internal_test.go
+++ b/backend/s3/s3_internal_test.go
@@ -248,6 +248,47 @@ func TestMergeDeleteMarkers(t *testing.T) {
}
}
+func TestRemoveAWSChunked(t *testing.T) {
+ ps := func(s string) *string {
+ return &s
+ }
+ tests := []struct {
+ name string
+ in *string
+ want *string
+ }{
+ {"nil", nil, nil},
+ {"empty", ps(""), nil},
+ {"only aws", ps("aws-chunked"), nil},
+ {"leading aws", ps("aws-chunked, gzip"), ps("gzip")},
+ {"trailing aws", ps("gzip, aws-chunked"), ps("gzip")},
+ {"middle aws", ps("gzip, aws-chunked, br"), ps("gzip,br")},
+ {"case insensitive", ps("GZip, AwS-ChUnKeD, Br"), ps("GZip,Br")},
+ {"duplicates", ps("aws-chunked , aws-chunked"), nil},
+ {"no aws normalize spaces", ps(" gzip , br "), ps(" gzip , br ")},
+ {"surrounding spaces", ps(" aws-chunked "), nil},
+ {"no change", ps("gzip, br"), ps("gzip, br")},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ got := removeAWSChunked(tc.in)
+ check := func(want, got *string) {
+ t.Helper()
+ if tc.want == nil {
+ assert.Nil(t, got)
+ } else {
+ require.NotNil(t, got)
+ assert.Equal(t, *tc.want, *got)
+ }
+ }
+ check(tc.want, got)
+ // Idempotent
+ got2 := removeAWSChunked(got)
+ check(got, got2)
+ })
+ }
+}
+
func (f *Fs) InternalTestVersions(t *testing.T) {
ctx := context.Background()