"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: let a failed CompleteMultipartUpload be retried
Nick Craig-Woodbe847e8be63f
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex da4450923..02564c331 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -903,19 +903,22 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj}defer up.endActivity()- // gofakes3 keeps its record of an upload whose completion fails, so- // forgetUpload removes that too, as the upload can't be retried.+ // A completion which fails before the commit leaves the upload as+ // it was, so the client can upload any missing parts and retry it,+ // as on S3. gofakes3 keeps its record of the upload too.if err := up.validate(input); err != nil {- b.forgetUpload(uploadID, up)return "", "", err}- // close commits the upload, failing with the upload left open if the- // streamed parts don't form the complete object; forgetUpload then- // tears it down. (After a successful or failed commit the abort it- // does is a no-op.)+ // close commits the upload, failing with ErrInvalidPart and the+ // upload left open if the streamed parts don't form the complete+ // object. Any other failure is of the commit, so the upload can't be+ // retried and forgetUpload tears it down. (After a failed commit the+ // abort it does is a no-op.)if err := up.close(); err != nil {- b.forgetUpload(uploadID, up)+ if !errors.Is(err, gofakes3.ErrInvalidPart) {+ b.forgetUpload(uploadID, up)+ }return "", "", err}diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex 125b0dce4..bc29be869 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -14,6 +14,7 @@ import ("math""net/url""path"+ "slices""strings""sync""sync/atomic"@@ -224,9 +225,10 @@ func TestMultipartOutOfOrder(t *testing.T) {}// TestMultipartNonContiguous checks that a multipart upload with a gap in the-// part numbers (which the in-order stream can't place) is rejected.+// part numbers (which the in-order stream can't place) is rejected, and+// can be completed once the gap is filled.func TestMultipartNonContiguous(t *testing.T) {- core, _, bucket := newMultipartTestServer(t, false)+ core, f, bucket := newMultipartTestServer(t, false)ctx := context.Background()const object = "gap.bin"@@ -234,14 +236,29 @@ func TestMultipartNonContiguous(t *testing.T) {require.NoError(t, err)var parts []minio.CompletePart- for _, pn := range []int{1, 2, 4} { // part 3 missing- data := []byte(random.String(40 * 1024))- p, err := core.PutObjectPart(ctx, bucket, object, uploadID, pn, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})+ var want []byte+ data := map[int][]byte{}+ putPart := func(pn int) {+ data[pn] = []byte(random.String(40 * 1024))+ p, err := core.PutObjectPart(ctx, bucket, object, uploadID, pn, bytes.NewReader(data[pn]), int64(len(data[pn])), minio.PutObjectPartOptions{})require.NoError(t, err)parts = append(parts, minio.CompletePart{PartNumber: pn, ETag: p.ETag})}+ for _, pn := range []int{1, 2, 4} { // part 3 missing+ putPart(pn)+ }_, err = core.CompleteMultipartUpload(ctx, bucket, object, uploadID, parts, minio.PutObjectOptions{})require.Error(t, err)++ // The upload is left open, so the missing part can be added+ putPart(3)+ slices.SortFunc(parts, func(a, b minio.CompletePart) int { return a.PartNumber - b.PartNumber })+ _, err = core.CompleteMultipartUpload(ctx, bucket, object, uploadID, parts, minio.PutObjectOptions{})+ require.NoError(t, err)+ for pn := 1; pn <= 4; pn++ {+ want = append(want, data[pn]...)+ }+ assert.Equal(t, want, readObject(t, f, bucket, object))}// requireOnly asserts that the bucket contains only the expected@@ -747,10 +764,10 @@ func TestMultipartReaper(t *testing.T) {requireOnly(t, f, bucket)}-// TestMultipartCompleteFailureForgotten checks that a completion which-// tears the upload down removes it from gofakes3's records too, so it-// isn't left in listings or memory.-func TestMultipartCompleteFailureForgotten(t *testing.T) {+// TestMultipartCompleteFailureRetry checks that a completion which+// fails with a wrong part list leaves the upload in place, so it can be+// retried with the right one, as on S3.+func TestMultipartCompleteFailureRetry(t *testing.T) {core, f, bucket := newMultipartTestServer(t, false)ctx := context.Background()const object = "bad-complete.bin"@@ -758,17 +775,22 @@ func TestMultipartCompleteFailureForgotten(t *testing.T) {uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})require.NoError(t, err)data := []byte(random.String(50 * 1024))- _, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})+ part, err := core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})require.NoError(t, err)_, err = core.CompleteMultipartUpload(ctx, bucket, object, uploadID, []minio.CompletePart{{PartNumber: 1, ETag: `"00000000000000000000000000000000"`}}, minio.PutObjectOptions{})require.Error(t, err)+ assert.Contains(t, err.Error(), "InvalidPart")// FIXME gofakes3 lists nothing when the delimiter is emptyuploads, err := core.ListMultipartUploads(ctx, bucket, "", "", "", "/", 1000)require.NoError(t, err)- assert.Empty(t, uploads.Uploads)- requireOnly(t, f, bucket)+ assert.Len(t, uploads.Uploads, 1)++ _, err = core.CompleteMultipartUpload(ctx, bucket, object, uploadID, []minio.CompletePart{{PartNumber: 1, ETag: part.ETag}}, minio.PutObjectOptions{})+ require.NoError(t, err)+ assert.Equal(t, data, readObject(t, f, bucket, object))+ requireOnly(t, f, bucket, object)}// TestMultipartReapExpiredUploads checks the reaper's rules directly: andiff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 929abcdde..2882c70fa 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -239,6 +239,11 @@ remote as if it had completed.been streamed, so the whole upload is failed and the client muststart it again. So is a failure in the stream to the remote itself. (The remote's own uploadstill retries its internal chunks.)+- A `CompleteMultipartUpload` which fails because the parts it lists+ don't match those uploaded, or there is a gap in the part numbers,+ leaves the upload open, as on S3, so the client can upload any+ missing parts and complete it again. If writing the completed object+ to the remote fails, the upload is failed and must be started again.- Parts are serialised into one stream, so ingest from the client iseffectively single-threaded. When streaming, the remote's own uploadruns concurrently with the parts arriving; with the local disk spool