"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: update gofakes3 to limit the memory requests can use
Nick Craig-Woodf4c1268f3316
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex deb2df2de..bcec5df4f 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -51,12 +51,6 @@ const multipartUploadPrefix = tempObjectPrefix + "multipart_"// buffer before the client is told to slow down and retry it.var multipartWaitTimeout = time.Minute-// errSlowDown asks the client to retry the request later.-//-// FIXME gofakes3 sends this with status 500 rather than the 503 S3 uses,-// though S3 clients retry either.-const errSlowDown gofakes3.ErrorCode = "SlowDown"-// multipartUpload tracks one in-flight S3 multipart upload. The parts are// written, in part-number order, into fh - a VFS file handle which either// streams straight through to the remote (the default) or is backed by the@@ -193,8 +187,8 @@ func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, er}// reserveUpload reserves a place for a new upload, failing with-// errSlowDown if --multipart-max-uploads are already in progress. The-// place is given back by releaseUpload or deleteUpload.+// gofakes3.ErrSlowDown if --multipart-max-uploads are already in progress.+// The place is given back by releaseUpload or deleteUpload.func (b *s3Backend) reserveUpload() error {n := b.uploads.Add(1)if maxUploads := int64(b.s.opt.MultipartMaxUploads); maxUploads > 0 && n > maxUploads {@@ -202,7 +196,7 @@ func (b *s3Backend) reserveUpload() error {b.warnMaxUploadsOnce.Do(func() {fs.Logf(nil, "serve s3: telling clients to slow down as --multipart-max-uploads %d multipart uploads are in progress", maxUploads)})- return errSlowDown+ return gofakes3.ErrSlowDown}return nil}@@ -250,7 +244,8 @@ func (b *s3Backend) deleteUpload(uploadID gofakes3.UploadID) {// The object the parts are written to is only created when the first part// arrives, so an upload with no parts holds no VFS or backend resources.// No more than --multipart-max-uploads can be in progress: after that-// errSlowDown is returned before anything is created on the remote.+// gofakes3.ErrSlowDown is returned before anything is created on the+// remote.func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objectName string, meta map[string]string) (gofakes3.UploadID, error) {_vfs, err := b.s.getVFS(ctx)if err != nil {@@ -484,9 +479,10 @@ const (// until it is the next part. Reserved bytes are returned// with release, or by the pump as the part is streamed.//-// A part which waits for longer than multipartWaitTimeout gets errSlowDown,-// asking the client to retry it later, so a waiting request isn't held open-// indefinitely. The wait also ends if ctx is cancelled.+// A part which waits for longer than multipartWaitTimeout gets+// gofakes3.ErrSlowDown, asking the client to retry it later, so a waiting+// request isn't held open indefinitely. The wait also ends if ctx is+// cancelled.//// size is the client-declared part length and is not trusted: a negative value// is rejected, and the admission test is written so a huge value can't overflow@@ -530,7 +526,7 @@ func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, sizecase <-ctx.Done():return 0, ctx.Err()case <-timer.C:- return 0, errSlowDown+ return 0, gofakes3.ErrSlowDown}}}@@ -733,7 +729,7 @@ func (up *multipartUpload) bufferPart(ctx context.Context, partNumber int, sizecase <-ctx.Done():err = ctx.Err()case <-timer.C:- err = errSlowDown+ err = gofakes3.ErrSlowDown}up.mu.Lock()if err != nil {diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex 2a6ddef53..e64a1d6c4 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -1211,7 +1211,7 @@ func TestMultipartBufferWaitGivesUp(t *testing.T) {require.ErrorIs(t, err, context.Canceled)_, err = b.UploadPart(ctx, bucket, object, uploadID, 3, 1, bytes.NewReader([]byte("3")))- assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)require.NoError(t, b.AbortMultipartUpload(ctx, bucket, object, uploadID))}@@ -1605,14 +1605,14 @@ func TestMultipartMaxUploads(t *testing.T) {id2, err := b.CreateMultipartUpload(ctx, bucket, "2", nil)require.NoError(t, err)_, err = b.CreateMultipartUpload(ctx, bucket, "3", nil)- assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)// Aborting an upload makes room.require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "1", id1))id3, err := b.CreateMultipartUpload(ctx, bucket, "3", nil)require.NoError(t, err)_, err = b.CreateMultipartUpload(ctx, bucket, "4", nil)- assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)// So does completing one._, _, err = b.CompleteMultipartUpload(ctx, bucket, "2", id2, &gofakes3.CompleteMultipartUploadRequest{})@@ -1622,7 +1622,7 @@ func TestMultipartMaxUploads(t *testing.T) {// A refused upload leaves nothing behind on the remote._, err = b.CreateMultipartUpload(ctx, bucket, "dir/deep/4", nil)- assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)_vfs, err := b.s.getVFS(ctx)require.NoError(t, err)_, err = _vfs.Stat(bucket + "/dir")diff --git a/go.mod b/go.modindex cbdca57db..2d99c8f9c 100644--- a/go.mod+++ b/go.mod@@ -74,7 +74,7 @@ require (github.com/quasilyte/go-ruleguard/dsl v0.3.23github.com/rclone/Proton-API-Bridge v1.0.5github.com/rclone/go-proton-api v1.0.4- github.com/rclone/gofakes3 v0.0.9+ github.com/rclone/gofakes3 v0.0.10-0.20260919171726-8edbb055e5ecgithub.com/rfjakob/eme v1.2.0github.com/rivo/uniseg v0.4.7github.com/rogpeppe/go-internal v1.15.0diff --git a/go.sum b/go.sumindex 9fc4ff90e..f602c442a 100644--- a/go.sum+++ b/go.sum@@ -494,8 +494,8 @@ github.com/rclone/Proton-API-Bridge v1.0.5 h1:K1++Qtk3PvgkiCCiv6Pahju1TMOzKY6VSwgithub.com/rclone/Proton-API-Bridge v1.0.5/go.mod h1:vCeOPhlXzevN0AFojgh1zsjhetiShy/ArvJ/xkFUDWk=github.com/rclone/go-proton-api v1.0.4 h1:AJW0e9pB4j0hVK4WqyGErFwaI+5MUQWPCtj5FYYxtPg=github.com/rclone/go-proton-api v1.0.4/go.mod h1:QAlkFfswzrBuxvCORWV8rZdddg52hahMN98CFWoFW1E=-github.com/rclone/gofakes3 v0.0.9 h1:1EzeGoSefXmk/DrmmJvJ6B9HR9uUMih4YB2fkQk88Wg=-github.com/rclone/gofakes3 v0.0.9/go.mod h1:0udb2hmIO9mHXDAgcdycCzSPPASAGIz410t+m0LKmnE=+github.com/rclone/gofakes3 v0.0.10-0.20260919171726-8edbb055e5ec h1:6JAnhUd7+U6wWrS14qpWnhNXWD/xtxNd8qVwyNvjflw=+github.com/rclone/gofakes3 v0.0.10-0.20260919171726-8edbb055e5ec/go.mod h1:0udb2hmIO9mHXDAgcdycCzSPPASAGIz410t+m0LKmnE=github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI=