"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
drive: reuse resumable-upload chunk buffers via multipart.NewRW
SillyZir393544b1162c
diff --git a/backend/drive/upload.go b/backend/drive/upload.goindex 10b3ab506..054024a7a 100644--- a/backend/drive/upload.go+++ b/backend/drive/upload.go@@ -11,7 +11,6 @@package driveimport (- "bytes""context""encoding/json""fmt"@@ -22,6 +21,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/readers""google.golang.org/api/drive/v3""google.golang.org/api/googleapi"@@ -112,8 +112,13 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,}// Make an http.Request for the range passed in+//+// The body is wrapped in readers.NoCloser so the transport can't upgrade it+// to an io.Closer and close it after each attempt — the chunk buffer is+// pool-backed, so an early close would free its pages and a retry would then+// read a dead buffer. The upload loop owns the buffer's lifetime.func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {- req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)+ req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))req.ContentLength = reqSizetotalSize := "*"if rx.ContentLength >= 0 {@@ -168,37 +173,40 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {start := int64(0)var StatusCode intvar err error- buf := make([]byte, int(rx.f.opt.ChunkSize))+ chunkSize := int64(rx.f.opt.ChunkSize)for finished := false; !finished; {- var reqSize int64- var chunk io.ReadSeeker+ reqSize := chunkSizeif rx.ContentLength >= 0 {- // If size known use repeatable reader for smoother bwlimitif start >= rx.ContentLength {break}- reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))- chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)- } else {- // If size unknown read into buffer- var n int- n, err = readers.ReadFill(rx.Media, buf)+ reqSize = min(rx.ContentLength-start, chunkSize)+ }++ // Buffer the chunk in memory from the global pool so reads are+ // repeatable for retries+ rw := multipart.NewRW()+ var n int64+ n, err = io.CopyN(rw, rx.Media, reqSize)+ if rx.ContentLength < 0 {if err == io.EOF {// Send the last chunk with the correct ContentLength// otherwise Google doesn't know we've finished- rx.ContentLength = start + int64(n)+ rx.ContentLength = start + nfinished = true- } else if err != nil {- return nil, err+ err = nil}- reqSize = int64(n)- chunk = bytes.NewReader(buf[:reqSize])+ reqSize = n+ }+ if err != nil {+ _ = rw.Close()+ return nil, err}// Transfer the chunkerr = rx.f.pacer.Call(func() (bool, error) {fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)- StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)+ StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)again, err := rx.f.shouldRetry(ctx, err)if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {again = false@@ -206,9 +214,13 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {}return again, err})+ closeErr := rw.Close()if err != nil {return nil, err}+ if closeErr != nil {+ return nil, closeErr+ }start += reqSize}diff --git a/backend/drive/upload_internal_test.go b/backend/drive/upload_internal_test.gonew file mode 100644index 000000000..9a90fe95d--- /dev/null+++ b/backend/drive/upload_internal_test.go@@ -0,0 +1,57 @@+package drive++import (+ "bytes"+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// TestResumableUploadRetry checks that a chunk which fails with a 5xx is+// retried successfully. The chunk buffer is pool-backed and implements+// io.Closer, so if it reaches the http transport unwrapped the transport+// closes it after the failed attempt, returning its pages to the pool, and+// the retry then reads a freed buffer.+func TestResumableUploadRetry(t *testing.T) {+ content := bytes.Repeat([]byte("resumable"), 512)+ var attempts atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ body, err := io.ReadAll(r.Body)+ require.NoError(t, err)+ if attempts.Add(1) == 1 {+ w.WriteHeader(http.StatusInternalServerError)+ return+ }+ assert.Equal(t, content, body, "retried chunk should re-send the full chunk")+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))+ }))+ defer server.Close()++ f := &Fs{+ pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),+ client: server.Client(),+ }+ f.opt.ChunkSize = fs.SizeSuffix(len(content))+ rx := &resumableUpload{+ f: f,+ remote: "remote",+ URI: server.URL,+ Media: bytes.NewReader(content),+ MediaType: "application/octet-stream",+ ContentLength: int64(len(content)),+ }+ info, err := rx.Upload(context.Background())+ require.NoError(t, err)+ require.NotNil(t, info)+ assert.Equal(t, "fake-id", info.Id)+ assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")+}