Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

s3api: report the stored checksum in GetObjectAttributes (#11529)

Javier Garcia8fdcf69eb0ed
diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml
index b40f8d3ea5..35ab1c1775 100644
--- a/.github/workflows/s3tests.yml
+++ b/.github/workflows/s3tests.yml
@@ -312,6 +312,7 @@ jobs:
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+ s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
s3tests/functional/test_s3.py::test_bucket_head \
s3tests/functional/test_s3.py::test_bucket_head_notexist \
s3tests/functional/test_s3.py::test_object_raw_authenticated \
@@ -1173,6 +1174,7 @@ jobs:
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+ s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
s3tests/functional/test_s3.py::test_bucket_head \
s3tests/functional/test_s3.py::test_bucket_head_notexist \
s3tests/functional/test_s3.py::test_object_raw_authenticated \
diff --git a/weed/s3api/s3api_object_handlers_attributes.go b/weed/s3api/s3api_object_handlers_attributes.go
index 4f372e2567..901409a26c 100644
--- a/weed/s3api/s3api_object_handlers_attributes.go
+++ b/weed/s3api/s3api_object_handlers_attributes.go
@@ -29,10 +29,8 @@ type GetObjectAttributesResponse struct {
// ObjectAttributesChecksum holds checksum info for GetObjectAttributes.
type ObjectAttributesChecksum struct {
- ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
- ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
- ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
- ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
+ ChecksumResult
+ ChecksumType string `xml:"ChecksumType,omitempty"`
}
// ObjectAttributesParts holds parts info for GetObjectAttributes.
@@ -77,6 +75,24 @@ func validateObjectAttributes(attrs map[string]struct{}) bool {
return true
}
+// objectAttributesChecksum returns the additional checksum that PutObject or
+// CompleteMultipartUpload stored with the object, or nil if it has none.
+func objectAttributesChecksum(entry *filer_pb.Entry) *ObjectAttributesChecksum {
+ if entry == nil || entry.Extended == nil {
+ return nil
+ }
+ value := string(entry.Extended[s3_constants.ExtChecksumValue])
+ if value == "" {
+ return nil
+ }
+ checksum := &ObjectAttributesChecksum{ChecksumType: string(entry.Extended[s3_constants.ExtChecksumType])}
+ checksum.SetChecksum(string(entry.Extended[s3_constants.ExtChecksumAlgorithm]), value)
+ if checksum.ChecksumResult == (ChecksumResult{}) {
+ return nil
+ }
+ return checksum
+}
+
func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
bucket, object := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("GetObjectAttributesHandler %s %s", bucket, object)
@@ -245,10 +261,9 @@ func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *htt
resp.StorageClass = storageClass
}
- // Checksum: accepted in validation so clients don't get a 400, but SeaweedFS
- // does not yet store S3 checksums (CRC32, CRC32C, SHA1, SHA256), so
- // resp.Checksum is intentionally left nil. When checksum storage is added,
- // populate resp.Checksum here.
+ if _, ok := requestedAttrs["Checksum"]; ok {
+ resp.Checksum = objectAttributesChecksum(entry)
+ }
if _, ok := requestedAttrs["ObjectSize"]; ok {
var size int64
diff --git a/weed/s3api/s3api_object_handlers_attributes_test.go b/weed/s3api/s3api_object_handlers_attributes_test.go
new file mode 100644
index 0000000000..6138c267e1
--- /dev/null
+++ b/weed/s3api/s3api_object_handlers_attributes_test.go
@@ -0,0 +1,88 @@
+package s3api
+
+import (
+ "encoding/xml"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// TestObjectAttributesChecksum verifies that GetObjectAttributes reports the
+// checksum that PutObject or CompleteMultipartUpload stored with the object
+func TestObjectAttributesChecksum(t *testing.T) {
+ testCases := []struct {
+ name string
+ extended map[string][]byte
+ want *ObjectAttributesChecksum
+ }{
+ {
+ name: "PutObject with SHA256",
+ extended: map[string][]byte{
+ s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumSHA256),
+ s3_constants.ExtChecksumValue: []byte("arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="),
+ },
+ want: &ObjectAttributesChecksum{
+ ChecksumResult: ChecksumResult{ChecksumSHA256: "arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="},
+ },
+ },
+ {
+ name: "multipart upload with a composite CRC32C",
+ extended: map[string][]byte{
+ s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC32C),
+ s3_constants.ExtChecksumValue: []byte("x3Y2bw==-3"),
+ s3_constants.ExtChecksumType: []byte("COMPOSITE"),
+ },
+ want: &ObjectAttributesChecksum{
+ ChecksumResult: ChecksumResult{ChecksumCRC32C: "x3Y2bw==-3"},
+ ChecksumType: "COMPOSITE",
+ },
+ },
+ {
+ name: "multipart upload with a full object CRC64NVME",
+ extended: map[string][]byte{
+ s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC64NVME),
+ s3_constants.ExtChecksumValue: []byte("AAAAAAAAAAA="),
+ s3_constants.ExtChecksumType: []byte("FULL_OBJECT"),
+ },
+ want: &ObjectAttributesChecksum{
+ ChecksumResult: ChecksumResult{ChecksumCRC64NVME: "AAAAAAAAAAA="},
+ ChecksumType: "FULL_OBJECT",
+ },
+ },
+ {
+ name: "no checksum",
+ extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},
+ },
+ {
+ name: "unknown algorithm",
+ extended: map[string][]byte{
+ s3_constants.ExtChecksumAlgorithm: []byte("X-Amz-Checksum-Md5"),
+ s3_constants.ExtChecksumValue: []byte("1B2M2Y8AsgTpgAmY7PhCfg=="),
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ assert.Equal(t, tc.want, objectAttributesChecksum(&filer_pb.Entry{Extended: tc.extended}))
+ })
+ }
+ assert.Nil(t, objectAttributesChecksum(&filer_pb.Entry{}))
+}
+
+// TestGetObjectAttributesChecksumXML verifies the Checksum element's layout
+func TestGetObjectAttributesChecksumXML(t *testing.T) {
+ resp := GetObjectAttributesResponse{
+ Checksum: &ObjectAttributesChecksum{
+ ChecksumResult: ChecksumResult{ChecksumCRC32: "NhCmhg=="},
+ ChecksumType: "FULL_OBJECT",
+ },
+ }
+ out, err := xml.Marshal(resp)
+ require.NoError(t, err)
+ assert.Equal(t, "<GetObjectAttributesResponse><Checksum><ChecksumCRC32>NhCmhg==</ChecksumCRC32>"+
+ "<ChecksumType>FULL_OBJECT</ChecksumType></Checksum></GetObjectAttributesResponse>", string(out))
+}