SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
s3api: do not store aws-chunked in an object's Content-Encoding (#11528)
Javier Garcia988fc4f7ba7b
diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.ymlindex 980b921ff4..b40f8d3ea5 100644--- a/.github/workflows/s3tests.yml+++ b/.github/workflows/s3tests.yml@@ -289,6 +289,7 @@ jobs:s3tests/functional/test_s3.py::test_object_write_check_etag \s3tests/functional/test_s3.py::test_object_write_cache_control \s3tests/functional/test_s3.py::test_object_write_expires \+ s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \s3tests/functional/test_s3.py::test_object_write_file \@@ -1149,6 +1150,7 @@ jobs:s3tests/functional/test_s3.py::test_object_write_check_etag \s3tests/functional/test_s3.py::test_object_write_cache_control \s3tests/functional/test_s3.py::test_object_write_expires \+ s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \s3tests/functional/test_s3.py::test_object_write_file \diff --git a/weed/s3api/s3_content_encoding_test.go b/weed/s3api/s3_content_encoding_test.goindex a50a8bb3c7..0b8f5cd9c6 100644--- a/weed/s3api/s3_content_encoding_test.go+++ b/weed/s3api/s3_content_encoding_test.go@@ -2,6 +2,7 @@ package s3apiimport ("bytes"+ "net/http""net/http/httptest""testing"@@ -202,3 +203,57 @@ func TestContentEncodingWithOtherHeaders(t *testing.T) {assert.Equal(t, "max-age=3600", getResp.Header().Get("Cache-Control"))assert.Equal(t, "attachment; filename=test.txt", getResp.Header().Get("Content-Disposition"))}++// TestContentEncodingDropsAwsChunked verifies that aws-chunked, the SigV4+// streaming framing of the request body, is not stored with the object, also+// when the encodings come in separate Content-Encoding fields+func TestContentEncodingDropsAwsChunked(t *testing.T) {+ testCases := []struct {+ name string+ fields []string+ stored string+ }{+ {"last", []string{"gzip, aws-chunked"}, "gzip"},+ {"first", []string{"aws-chunked, gzip"}, "gzip"},+ {"no spaces", []string{"aws-chunked,gzip,br"}, "gzip, br"},+ {"alone", []string{"aws-chunked"}, ""},+ {"capitals", []string{"AWS-Chunked"}, ""},+ {"twice", []string{"aws-chunked, aws-chunked"}, ""},+ {"separate fields", []string{"aws-chunked", "gzip"}, "gzip"},+ {"separate fields, alone", []string{"aws-chunked", "aws-chunked"}, ""},+ {"without aws-chunked", []string{"deflate, gzip"}, "deflate, gzip"},+ {"without aws-chunked, separate fields", []string{"deflate", "gzip"}, "deflate, gzip"},+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ assert.Equal(t, tc.stored, storedContentEncoding(tc.fields))++ // CreateMultipartUpload+ putReq := httptest.NewRequest("PUT", "/test-bucket/test-object.txt", bytes.NewBufferString("body"))+ for _, field := range tc.fields {+ putReq.Header.Add("Content-Encoding", field)+ }+ metadata, errCode := ParseS3Metadata(putReq, nil, false)+ require.Equal(t, 0, int(errCode))+ if tc.stored == "" {+ assert.NotContains(t, metadata, "Content-Encoding")+ } else {+ assert.Equal(t, []byte(tc.stored), metadata["Content-Encoding"])+ }++ // CopyObject with the REPLACE metadata directive+ copyReq := http.Header{}+ for _, field := range tc.fields {+ copyReq.Add("Content-Encoding", field)+ }+ metadata, err := processMetadataBytes(copyReq, map[string][]byte{"Content-Encoding": []byte("br")}, true, false)+ require.NoError(t, err)+ if tc.stored == "" {+ assert.NotContains(t, metadata, "Content-Encoding")+ } else {+ assert.Equal(t, []byte(tc.stored), metadata["Content-Encoding"])+ }+ })+ }+}diff --git a/weed/s3api/s3_metadata_util.go b/weed/s3api/s3_metadata_util.goindex 71ef623f9c..6f71776f1e 100644--- a/weed/s3api/s3_metadata_util.go+++ b/weed/s3api/s3_metadata_util.go@@ -30,7 +30,7 @@ func ParseS3Metadata(r *http.Request, existing map[string][]byte, isReplace bool}// Content-Encoding (standard HTTP header used by S3)- if ce := r.Header.Get("Content-Encoding"); ce != "" {+ if ce := storedContentEncoding(r.Header.Values("Content-Encoding")); ce != "" {metadata["Content-Encoding"] = []byte(ce)}@@ -108,3 +108,28 @@ func ParseS3Metadata(r *http.Request, existing map[string][]byte, isReplace boolreturn metadata, s3err.ErrNone}++// storedContentEncoding returns the Content-Encoding to keep with an object,+// from the values of the request's Content-Encoding fields, which it combines+// as one list. aws-chunked names the SigV4 streaming framing of the request+// body, which is decoded on upload, so S3 does not store it: "gzip,+// aws-chunked" is kept as "gzip", and "aws-chunked" alone as no+// Content-Encoding at all.+func storedContentEncoding(values []string) string {+ value := strings.Join(values, ", ")+ var kept []string+ chunked := false+ for _, encoding := range strings.Split(value, ",") {+ encoding = strings.TrimSpace(encoding)+ switch {+ case strings.EqualFold(encoding, "aws-chunked"):+ chunked = true+ case encoding != "":+ kept = append(kept, encoding)+ }+ }+ if !chunked {+ return value+ }+ return strings.Join(kept, ", ")+}diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.goindex 222b8c6a40..6f6cbf3ec0 100644--- a/weed/s3api/s3api_object_handlers_copy.go+++ b/weed/s3api/s3api_object_handlers_copy.go@@ -1197,7 +1197,11 @@ func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, rep}}for _, h := range copyReplaceSystemHeaders {- if v := reqHeader.Get(h); v != "" {+ v := reqHeader.Get(h)+ if h == "Content-Encoding" {+ v = storedContentEncoding(reqHeader.Values(h))+ }+ if v != "" {metadata[h] = []byte(v)}}diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.goindex 53f7fe0274..eeea0a7365 100644--- a/weed/s3api/s3api_object_handlers_put.go+++ b/weed/s3api/s3api_object_handlers_put.go@@ -833,8 +833,12 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReaderentry.Extended[k] = []byte(v[0])} else {switch k {- case "Cache-Control", "Expires", "Content-Disposition", "Content-Encoding", "Content-Language":+ case "Cache-Control", "Expires", "Content-Disposition", "Content-Language":entry.Extended[k] = []byte(v[0])+ case "Content-Encoding":+ if ce := storedContentEncoding(v); ce != "" {+ entry.Extended[k] = []byte(ce)+ }}}if k == "Response-Content-Disposition" {