SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
s3api: report the stored checksum in GetObjectAttributes (#11529)
Javier Garcia8fdcf69eb0ed
diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.ymlindex b40f8d3ea5..35ab1c1775 100644--- a/.github/workflows/s3tests.yml+++ b/.github/workflows/s3tests.yml@@ -312,6 +312,7 @@ jobs:s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \+ s3tests/functional/test_s3.py::test_get_checksum_object_attributes \s3tests/functional/test_s3.py::test_bucket_head \s3tests/functional/test_s3.py::test_bucket_head_notexist \s3tests/functional/test_s3.py::test_object_raw_authenticated \@@ -1173,6 +1174,7 @@ jobs:s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \+ s3tests/functional/test_s3.py::test_get_checksum_object_attributes \s3tests/functional/test_s3.py::test_bucket_head \s3tests/functional/test_s3.py::test_bucket_head_notexist \s3tests/functional/test_s3.py::test_object_raw_authenticated \diff --git a/weed/s3api/s3api_object_handlers_attributes.go b/weed/s3api/s3api_object_handlers_attributes.goindex 4f372e2567..901409a26c 100644--- a/weed/s3api/s3api_object_handlers_attributes.go+++ b/weed/s3api/s3api_object_handlers_attributes.go@@ -29,10 +29,8 @@ type GetObjectAttributesResponse struct {// ObjectAttributesChecksum holds checksum info for GetObjectAttributes.type ObjectAttributesChecksum struct {- ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`- ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`- ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`- ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`+ ChecksumResult+ ChecksumType string `xml:"ChecksumType,omitempty"`}// ObjectAttributesParts holds parts info for GetObjectAttributes.@@ -77,6 +75,24 @@ func validateObjectAttributes(attrs map[string]struct{}) bool {return true}+// objectAttributesChecksum returns the additional checksum that PutObject or+// CompleteMultipartUpload stored with the object, or nil if it has none.+func objectAttributesChecksum(entry *filer_pb.Entry) *ObjectAttributesChecksum {+ if entry == nil || entry.Extended == nil {+ return nil+ }+ value := string(entry.Extended[s3_constants.ExtChecksumValue])+ if value == "" {+ return nil+ }+ checksum := &ObjectAttributesChecksum{ChecksumType: string(entry.Extended[s3_constants.ExtChecksumType])}+ checksum.SetChecksum(string(entry.Extended[s3_constants.ExtChecksumAlgorithm]), value)+ if checksum.ChecksumResult == (ChecksumResult{}) {+ return nil+ }+ return checksum+}+func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {bucket, object := s3_constants.GetBucketAndObject(r)glog.V(3).Infof("GetObjectAttributesHandler %s %s", bucket, object)@@ -245,10 +261,9 @@ func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *httresp.StorageClass = storageClass}- // Checksum: accepted in validation so clients don't get a 400, but SeaweedFS- // does not yet store S3 checksums (CRC32, CRC32C, SHA1, SHA256), so- // resp.Checksum is intentionally left nil. When checksum storage is added,- // populate resp.Checksum here.+ if _, ok := requestedAttrs["Checksum"]; ok {+ resp.Checksum = objectAttributesChecksum(entry)+ }if _, ok := requestedAttrs["ObjectSize"]; ok {var size int64diff --git a/weed/s3api/s3api_object_handlers_attributes_test.go b/weed/s3api/s3api_object_handlers_attributes_test.gonew file mode 100644index 0000000000..6138c267e1--- /dev/null+++ b/weed/s3api/s3api_object_handlers_attributes_test.go@@ -0,0 +1,88 @@+package s3api++import (+ "encoding/xml"+ "testing"++ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// TestObjectAttributesChecksum verifies that GetObjectAttributes reports the+// checksum that PutObject or CompleteMultipartUpload stored with the object+func TestObjectAttributesChecksum(t *testing.T) {+ testCases := []struct {+ name string+ extended map[string][]byte+ want *ObjectAttributesChecksum+ }{+ {+ name: "PutObject with SHA256",+ extended: map[string][]byte{+ s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumSHA256),+ s3_constants.ExtChecksumValue: []byte("arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="),+ },+ want: &ObjectAttributesChecksum{+ ChecksumResult: ChecksumResult{ChecksumSHA256: "arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="},+ },+ },+ {+ name: "multipart upload with a composite CRC32C",+ extended: map[string][]byte{+ s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC32C),+ s3_constants.ExtChecksumValue: []byte("x3Y2bw==-3"),+ s3_constants.ExtChecksumType: []byte("COMPOSITE"),+ },+ want: &ObjectAttributesChecksum{+ ChecksumResult: ChecksumResult{ChecksumCRC32C: "x3Y2bw==-3"},+ ChecksumType: "COMPOSITE",+ },+ },+ {+ name: "multipart upload with a full object CRC64NVME",+ extended: map[string][]byte{+ s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC64NVME),+ s3_constants.ExtChecksumValue: []byte("AAAAAAAAAAA="),+ s3_constants.ExtChecksumType: []byte("FULL_OBJECT"),+ },+ want: &ObjectAttributesChecksum{+ ChecksumResult: ChecksumResult{ChecksumCRC64NVME: "AAAAAAAAAAA="},+ ChecksumType: "FULL_OBJECT",+ },+ },+ {+ name: "no checksum",+ extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},+ },+ {+ name: "unknown algorithm",+ extended: map[string][]byte{+ s3_constants.ExtChecksumAlgorithm: []byte("X-Amz-Checksum-Md5"),+ s3_constants.ExtChecksumValue: []byte("1B2M2Y8AsgTpgAmY7PhCfg=="),+ },+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ assert.Equal(t, tc.want, objectAttributesChecksum(&filer_pb.Entry{Extended: tc.extended}))+ })+ }+ assert.Nil(t, objectAttributesChecksum(&filer_pb.Entry{}))+}++// TestGetObjectAttributesChecksumXML verifies the Checksum element's layout+func TestGetObjectAttributesChecksumXML(t *testing.T) {+ resp := GetObjectAttributesResponse{+ Checksum: &ObjectAttributesChecksum{+ ChecksumResult: ChecksumResult{ChecksumCRC32: "NhCmhg=="},+ ChecksumType: "FULL_OBJECT",+ },+ }+ out, err := xml.Marshal(resp)+ require.NoError(t, err)+ assert.Equal(t, "<GetObjectAttributesResponse><Checksum><ChecksumCRC32>NhCmhg==</ChecksumCRC32>"++ "<ChecksumType>FULL_OBJECT</ChecksumType></Checksum></GetObjectAttributesResponse>", string(out))+}