Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: let a failed CompleteMultipartUpload be retried

Nick Craig-Woodbe847e8be63f
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index da4450923..02564c331 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -903,19 +903,22 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
}
defer up.endActivity()
- // gofakes3 keeps its record of an upload whose completion fails, so
- // forgetUpload removes that too, as the upload can't be retried.
+ // A completion which fails before the commit leaves the upload as
+ // it was, so the client can upload any missing parts and retry it,
+ // as on S3. gofakes3 keeps its record of the upload too.
if err := up.validate(input); err != nil {
- b.forgetUpload(uploadID, up)
return "", "", err
}
- // close commits the upload, failing with the upload left open if the
- // streamed parts don't form the complete object; forgetUpload then
- // tears it down. (After a successful or failed commit the abort it
- // does is a no-op.)
+ // close commits the upload, failing with ErrInvalidPart and the
+ // upload left open if the streamed parts don't form the complete
+ // object. Any other failure is of the commit, so the upload can't be
+ // retried and forgetUpload tears it down. (After a failed commit the
+ // abort it does is a no-op.)
if err := up.close(); err != nil {
- b.forgetUpload(uploadID, up)
+ if !errors.Is(err, gofakes3.ErrInvalidPart) {
+ b.forgetUpload(uploadID, up)
+ }
return "", "", err
}
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index 125b0dce4..bc29be869 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -14,6 +14,7 @@ import (
"math"
"net/url"
"path"
+ "slices"
"strings"
"sync"
"sync/atomic"
@@ -224,9 +225,10 @@ func TestMultipartOutOfOrder(t *testing.T) {
}
// TestMultipartNonContiguous checks that a multipart upload with a gap in the
-// part numbers (which the in-order stream can't place) is rejected.
+// part numbers (which the in-order stream can't place) is rejected, and
+// can be completed once the gap is filled.
func TestMultipartNonContiguous(t *testing.T) {
- core, _, bucket := newMultipartTestServer(t, false)
+ core, f, bucket := newMultipartTestServer(t, false)
ctx := context.Background()
const object = "gap.bin"
@@ -234,14 +236,29 @@ func TestMultipartNonContiguous(t *testing.T) {
require.NoError(t, err)
var parts []minio.CompletePart
- for _, pn := range []int{1, 2, 4} { // part 3 missing
- data := []byte(random.String(40 * 1024))
- p, err := core.PutObjectPart(ctx, bucket, object, uploadID, pn, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
+ var want []byte
+ data := map[int][]byte{}
+ putPart := func(pn int) {
+ data[pn] = []byte(random.String(40 * 1024))
+ p, err := core.PutObjectPart(ctx, bucket, object, uploadID, pn, bytes.NewReader(data[pn]), int64(len(data[pn])), minio.PutObjectPartOptions{})
require.NoError(t, err)
parts = append(parts, minio.CompletePart{PartNumber: pn, ETag: p.ETag})
}
+ for _, pn := range []int{1, 2, 4} { // part 3 missing
+ putPart(pn)
+ }
_, err = core.CompleteMultipartUpload(ctx, bucket, object, uploadID, parts, minio.PutObjectOptions{})
require.Error(t, err)
+
+ // The upload is left open, so the missing part can be added
+ putPart(3)
+ slices.SortFunc(parts, func(a, b minio.CompletePart) int { return a.PartNumber - b.PartNumber })
+ _, err = core.CompleteMultipartUpload(ctx, bucket, object, uploadID, parts, minio.PutObjectOptions{})
+ require.NoError(t, err)
+ for pn := 1; pn <= 4; pn++ {
+ want = append(want, data[pn]...)
+ }
+ assert.Equal(t, want, readObject(t, f, bucket, object))
}
// requireOnly asserts that the bucket contains only the expected
@@ -747,10 +764,10 @@ func TestMultipartReaper(t *testing.T) {
requireOnly(t, f, bucket)
}
-// TestMultipartCompleteFailureForgotten checks that a completion which
-// tears the upload down removes it from gofakes3's records too, so it
-// isn't left in listings or memory.
-func TestMultipartCompleteFailureForgotten(t *testing.T) {
+// TestMultipartCompleteFailureRetry checks that a completion which
+// fails with a wrong part list leaves the upload in place, so it can be
+// retried with the right one, as on S3.
+func TestMultipartCompleteFailureRetry(t *testing.T) {
core, f, bucket := newMultipartTestServer(t, false)
ctx := context.Background()
const object = "bad-complete.bin"
@@ -758,17 +775,22 @@ func TestMultipartCompleteFailureForgotten(t *testing.T) {
uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})
require.NoError(t, err)
data := []byte(random.String(50 * 1024))
- _, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
+ part, err := core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
require.NoError(t, err)
_, err = core.CompleteMultipartUpload(ctx, bucket, object, uploadID, []minio.CompletePart{{PartNumber: 1, ETag: `"00000000000000000000000000000000"`}}, minio.PutObjectOptions{})
require.Error(t, err)
+ assert.Contains(t, err.Error(), "InvalidPart")
// FIXME gofakes3 lists nothing when the delimiter is empty
uploads, err := core.ListMultipartUploads(ctx, bucket, "", "", "", "/", 1000)
require.NoError(t, err)
- assert.Empty(t, uploads.Uploads)
- requireOnly(t, f, bucket)
+ assert.Len(t, uploads.Uploads, 1)
+
+ _, err = core.CompleteMultipartUpload(ctx, bucket, object, uploadID, []minio.CompletePart{{PartNumber: 1, ETag: part.ETag}}, minio.PutObjectOptions{})
+ require.NoError(t, err)
+ assert.Equal(t, data, readObject(t, f, bucket, object))
+ requireOnly(t, f, bucket, object)
}
// TestMultipartReapExpiredUploads checks the reaper's rules directly: an
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 929abcdde..2882c70fa 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -239,6 +239,11 @@ remote as if it had completed.
been streamed, so the whole upload is failed and the client must
start it again. So is a failure in the stream to the remote itself. (The remote's own upload
still retries its internal chunks.)
+- A `CompleteMultipartUpload` which fails because the parts it lists
+ don't match those uploaded, or there is a gap in the part numbers,
+ leaves the upload open, as on S3, so the client can upload any
+ missing parts and complete it again. If writing the completed object
+ to the remote fails, the upload is failed and must be started again.
- Parts are serialised into one stream, so ingest from the client is
effectively single-threaded. When streaming, the remote's own upload
runs concurrently with the parts arriving; with the local disk spool