Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: update gofakes3 to limit the memory requests can use

Nick Craig-Woodf4c1268f3316
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index deb2df2de..bcec5df4f 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -51,12 +51,6 @@ const multipartUploadPrefix = tempObjectPrefix + "multipart_"
// buffer before the client is told to slow down and retry it.
var multipartWaitTimeout = time.Minute
-// errSlowDown asks the client to retry the request later.
-//
-// FIXME gofakes3 sends this with status 500 rather than the 503 S3 uses,
-// though S3 clients retry either.
-const errSlowDown gofakes3.ErrorCode = "SlowDown"
-
// multipartUpload tracks one in-flight S3 multipart upload. The parts are
// written, in part-number order, into fh - a VFS file handle which either
// streams straight through to the remote (the default) or is backed by the
@@ -193,8 +187,8 @@ func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, er
}
// reserveUpload reserves a place for a new upload, failing with
-// errSlowDown if --multipart-max-uploads are already in progress. The
-// place is given back by releaseUpload or deleteUpload.
+// gofakes3.ErrSlowDown if --multipart-max-uploads are already in progress.
+// The place is given back by releaseUpload or deleteUpload.
func (b *s3Backend) reserveUpload() error {
n := b.uploads.Add(1)
if maxUploads := int64(b.s.opt.MultipartMaxUploads); maxUploads > 0 && n > maxUploads {
@@ -202,7 +196,7 @@ func (b *s3Backend) reserveUpload() error {
b.warnMaxUploadsOnce.Do(func() {
fs.Logf(nil, "serve s3: telling clients to slow down as --multipart-max-uploads %d multipart uploads are in progress", maxUploads)
})
- return errSlowDown
+ return gofakes3.ErrSlowDown
}
return nil
}
@@ -250,7 +244,8 @@ func (b *s3Backend) deleteUpload(uploadID gofakes3.UploadID) {
// The object the parts are written to is only created when the first part
// arrives, so an upload with no parts holds no VFS or backend resources.
// No more than --multipart-max-uploads can be in progress: after that
-// errSlowDown is returned before anything is created on the remote.
+// gofakes3.ErrSlowDown is returned before anything is created on the
+// remote.
func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objectName string, meta map[string]string) (gofakes3.UploadID, error) {
_vfs, err := b.s.getVFS(ctx)
if err != nil {
@@ -484,9 +479,10 @@ const (
// until it is the next part. Reserved bytes are returned
// with release, or by the pump as the part is streamed.
//
-// A part which waits for longer than multipartWaitTimeout gets errSlowDown,
-// asking the client to retry it later, so a waiting request isn't held open
-// indefinitely. The wait also ends if ctx is cancelled.
+// A part which waits for longer than multipartWaitTimeout gets
+// gofakes3.ErrSlowDown, asking the client to retry it later, so a waiting
+// request isn't held open indefinitely. The wait also ends if ctx is
+// cancelled.
//
// size is the client-declared part length and is not trusted: a negative value
// is rejected, and the admission test is written so a huge value can't overflow
@@ -530,7 +526,7 @@ func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size
case <-ctx.Done():
return 0, ctx.Err()
case <-timer.C:
- return 0, errSlowDown
+ return 0, gofakes3.ErrSlowDown
}
}
}
@@ -733,7 +729,7 @@ func (up *multipartUpload) bufferPart(ctx context.Context, partNumber int, size
case <-ctx.Done():
err = ctx.Err()
case <-timer.C:
- err = errSlowDown
+ err = gofakes3.ErrSlowDown
}
up.mu.Lock()
if err != nil {
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index 2a6ddef53..e64a1d6c4 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -1211,7 +1211,7 @@ func TestMultipartBufferWaitGivesUp(t *testing.T) {
require.ErrorIs(t, err, context.Canceled)
_, err = b.UploadPart(ctx, bucket, object, uploadID, 3, 1, bytes.NewReader([]byte("3")))
- assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)
require.NoError(t, b.AbortMultipartUpload(ctx, bucket, object, uploadID))
}
@@ -1605,14 +1605,14 @@ func TestMultipartMaxUploads(t *testing.T) {
id2, err := b.CreateMultipartUpload(ctx, bucket, "2", nil)
require.NoError(t, err)
_, err = b.CreateMultipartUpload(ctx, bucket, "3", nil)
- assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)
// Aborting an upload makes room.
require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "1", id1))
id3, err := b.CreateMultipartUpload(ctx, bucket, "3", nil)
require.NoError(t, err)
_, err = b.CreateMultipartUpload(ctx, bucket, "4", nil)
- assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)
// So does completing one.
_, _, err = b.CompleteMultipartUpload(ctx, bucket, "2", id2, &gofakes3.CompleteMultipartUploadRequest{})
@@ -1622,7 +1622,7 @@ func TestMultipartMaxUploads(t *testing.T) {
// A refused upload leaves nothing behind on the remote.
_, err = b.CreateMultipartUpload(ctx, bucket, "dir/deep/4", nil)
- assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)
_vfs, err := b.s.getVFS(ctx)
require.NoError(t, err)
_, err = _vfs.Stat(bucket + "/dir")
diff --git a/go.mod b/go.mod
index cbdca57db..2d99c8f9c 100644
--- a/go.mod
+++ b/go.mod
@@ -74,7 +74,7 @@ require (
github.com/quasilyte/go-ruleguard/dsl v0.3.23
github.com/rclone/Proton-API-Bridge v1.0.5
github.com/rclone/go-proton-api v1.0.4
- github.com/rclone/gofakes3 v0.0.9
+ github.com/rclone/gofakes3 v0.0.10-0.20260919171726-8edbb055e5ec
github.com/rfjakob/eme v1.2.0
github.com/rivo/uniseg v0.4.7
github.com/rogpeppe/go-internal v1.15.0
diff --git a/go.sum b/go.sum
index 9fc4ff90e..f602c442a 100644
--- a/go.sum
+++ b/go.sum
@@ -494,8 +494,8 @@ github.com/rclone/Proton-API-Bridge v1.0.5 h1:K1++Qtk3PvgkiCCiv6Pahju1TMOzKY6VSw
github.com/rclone/Proton-API-Bridge v1.0.5/go.mod h1:vCeOPhlXzevN0AFojgh1zsjhetiShy/ArvJ/xkFUDWk=
github.com/rclone/go-proton-api v1.0.4 h1:AJW0e9pB4j0hVK4WqyGErFwaI+5MUQWPCtj5FYYxtPg=
github.com/rclone/go-proton-api v1.0.4/go.mod h1:QAlkFfswzrBuxvCORWV8rZdddg52hahMN98CFWoFW1E=
-github.com/rclone/gofakes3 v0.0.9 h1:1EzeGoSefXmk/DrmmJvJ6B9HR9uUMih4YB2fkQk88Wg=
-github.com/rclone/gofakes3 v0.0.9/go.mod h1:0udb2hmIO9mHXDAgcdycCzSPPASAGIz410t+m0LKmnE=
+github.com/rclone/gofakes3 v0.0.10-0.20260919171726-8edbb055e5ec h1:6JAnhUd7+U6wWrS14qpWnhNXWD/xtxNd8qVwyNvjflw=
+github.com/rclone/gofakes3 v0.0.10-0.20260919171726-8edbb055e5ec/go.mod h1:0udb2hmIO9mHXDAgcdycCzSPPASAGIz410t+m0LKmnE=
github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=
github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI=