SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
s3: populate Initiated timestamp in ListMultipartUploads (#11313)
Chris Lu0e82b4e3513b
diff --git a/test/s3/normal/s3_list_multipart_uploads_test.go b/test/s3/normal/s3_list_multipart_uploads_test.gonew file mode 100644index 0000000000..fe1ca7edc5--- /dev/null+++ b/test/s3/normal/s3_list_multipart_uploads_test.go@@ -0,0 +1,66 @@+package example++import (+ "testing"+ "time"++ "github.com/aws/aws-sdk-go/aws"+ "github.com/aws/aws-sdk-go/service/s3"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// TestListMultipartUploadsInitiated verifies that ListMultipartUploads+// returns the Initiated timestamp for each in-progress upload, and that+// the timestamp is preserved across repeated listings rather than+// reflecting the time of the listing.+func TestListMultipartUploadsInitiated(t *testing.T) {+ if testing.Short() {+ t.Skip("Skipping integration test in short mode")+ }++ cluster, err := startMiniCluster(t)+ require.NoError(t, err)+ defer cluster.Stop()++ bucket := createTestBucket(t, cluster, "test-list-mpu-initiated-")++ beforeInit := time.Now().UTC()+ createOut, err := cluster.s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{+ Bucket: aws.String(bucket),+ Key: aws.String("unfinished.bin"),+ })+ require.NoError(t, err)+ afterInit := time.Now().UTC()++ listOut, err := cluster.s3Client.ListMultipartUploads(&s3.ListMultipartUploadsInput{+ Bucket: aws.String(bucket),+ })+ require.NoError(t, err)+ require.Len(t, listOut.Uploads, 1)++ upload := listOut.Uploads[0]+ assert.Equal(t, "unfinished.bin", aws.StringValue(upload.Key))+ assert.Equal(t, aws.StringValue(createOut.UploadId), aws.StringValue(upload.UploadId))+ require.NotNil(t, upload.Initiated, "Initiated timestamp must be populated")++ initiated := upload.Initiated.UTC()+ assert.False(t, initiated.Before(beforeInit.Add(-time.Second)), "Initiated %v is before upload creation %v", initiated, beforeInit)+ assert.False(t, initiated.After(afterInit.Add(time.Second)), "Initiated %v is after upload creation %v", initiated, afterInit)++ time.Sleep(2 * time.Second)+ listOut2, err := cluster.s3Client.ListMultipartUploads(&s3.ListMultipartUploadsInput{+ Bucket: aws.String(bucket),+ })+ require.NoError(t, err)+ require.Len(t, listOut2.Uploads, 1)+ require.NotNil(t, listOut2.Uploads[0].Initiated, "Initiated timestamp must be populated on repeated listing")+ assert.True(t, listOut2.Uploads[0].Initiated.Equal(*upload.Initiated), "Initiated must be preserved across listings, got %v then %v", *upload.Initiated, *listOut2.Uploads[0].Initiated)++ _, err = cluster.s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{+ Bucket: aws.String(bucket),+ Key: aws.String("unfinished.bin"),+ UploadId: createOut.UploadId,+ })+ require.NoError(t, err)+}diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.goindex 0ea4c29f0c..33792dc2c7 100644--- a/weed/s3api/filer_multipart.go+++ b/weed/s3api/filer_multipart.go@@ -1072,10 +1072,14 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInputif *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) {continue}- output.Upload = append(output.Upload, &s3.MultipartUpload{+ upload := &s3.MultipartUpload{Key: objectKey(aws.String(key)),UploadId: aws.String(entry.Name),- })+ }+ if entry.Attributes != nil {+ upload.Initiated = aws.Time(time.Unix(entry.Attributes.Crtime, int64(entry.Attributes.CrtimeNs)))+ }+ output.Upload = append(output.Upload, upload)uploadsCount += 1}if uploadsCount >= *input.MaxUploads {