Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

s3: UploadPart inherits the checksum algorithm of its multipart upload

Chris Luf15b98097633
diff --git a/test/s3/checksum/issue11401_repro_test.go b/test/s3/checksum/issue11401_repro_test.go
new file mode 100644
index 0000000000..f6ee2e49b2
--- /dev/null
+++ b/test/s3/checksum/issue11401_repro_test.go
@@ -0,0 +1,85 @@
+package checksum_test
+
+import (
+ "bytes"
+ "context"
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+ "github.com/stretchr/testify/require"
+)
+
+// newWhenRequiredChecksumClient returns a client that only sends checksum
+// headers when the request asks for them, so tests can upload parts with no
+// checksum headers at all.
+func newWhenRequiredChecksumClient(t *testing.T) *s3.Client {
+ t.Helper()
+
+ cfg, err := config.LoadDefaultConfig(context.TODO(),
+ config.WithRegion(defaultConfig.Region),
+ config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+ defaultConfig.AccessKey, defaultConfig.SecretKey, "")),
+ config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+ func(service, region string, _ ...interface{}) (aws.Endpoint, error) {
+ return aws.Endpoint{
+ URL: defaultConfig.Endpoint,
+ SigningRegion: defaultConfig.Region,
+ HostnameImmutable: true,
+ }, nil
+ })),
+ )
+ require.NoError(t, err)
+ return s3.NewFromConfig(cfg, func(o *s3.Options) {
+ o.UsePathStyle = true
+ o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+ })
+}
+
+// UploadPart without checksum headers inherits the algorithm declared at
+// CreateMultipartUpload, so the part still gets a checksum and the upload can
+// be completed with it (AWS behavior).
+func TestMultipartPartInheritsChecksumAlgorithm(t *testing.T) {
+ client := newWhenRequiredChecksumClient(t)
+
+ bucket := uniqueBucket()
+ createBucket(t, client, bucket)
+ defer cleanupBucket(t, client, bucket)
+
+ body := bytes.Repeat([]byte("y"), 1024)
+ key := "inherit-algo"
+
+ create, err := client.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(key),
+ ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
+ })
+ require.NoError(t, err)
+
+ part, err := client.UploadPart(context.Background(), &s3.UploadPartInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(key),
+ UploadId: create.UploadId,
+ PartNumber: aws.Int32(1),
+ Body: bytes.NewReader(body),
+ })
+ require.NoError(t, err)
+ require.NotEmpty(t, aws.ToString(part.ChecksumSHA256))
+
+ done, err := client.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(key),
+ UploadId: create.UploadId,
+ MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{{
+ ETag: part.ETag,
+ PartNumber: aws.Int32(1),
+ ChecksumSHA256: part.ChecksumSHA256,
+ }}},
+ })
+ require.NoError(t, err)
+ require.Equal(t, types.ChecksumTypeComposite, done.ChecksumType)
+ require.NotEmpty(t, aws.ToString(done.ChecksumSHA256))
+}
diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go
index 6926e3da01..9a7fe7b8e2 100644
--- a/weed/s3api/s3api_object_handlers_multipart.go
+++ b/weed/s3api/s3api_object_handlers_multipart.go
@@ -445,6 +445,16 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
}
}
+ // Parts inherit the checksum algorithm declared at CreateMultipartUpload
+ // when the request doesn't specify one (AWS behavior).
+ if headerName := string(uploadEntry.Extended[s3_constants.ExtChecksumAlgorithm]); headerName != "" {
+ if algo, _, code := detectRequestedChecksumAlgorithm(r); code == s3err.ErrNone && algo == ChecksumAlgorithmNone {
+ if name := checksumAlgorithmNameFromHeaderName(headerName); name != "" {
+ r.Header.Set(s3_constants.AmzChecksumAlgorithm, name)
+ }
+ }
+ }
+
filePath := s3a.genPartUploadPath(bucket, uploadID, partID)
if partID == 1 && r.Header.Get("Content-Type") == "" {