"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
dropbox: reuse upload chunk buffers via the global memory pool
Nick Craig-Wood738eadf8dfe5
diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.goindex e69ff17ce..826dee6dd 100644--- a/backend/dropbox/dropbox.go+++ b/backend/dropbox/dropbox.go@@ -51,6 +51,7 @@ import ("github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/lib/batcher""github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/oauthutil""github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/readers"@@ -2079,11 +2080,6 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f// write chunksin := readers.NewCountingReader(in0)- bufSize := chunkSize- if size >= 0 && size < bufSize {- bufSize = size- }- buf := make([]byte, int(bufSize))cursor := files.UploadSessionCursor{SessionId: res.SessionId,Offset: 0,@@ -2103,14 +2099,21 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *ffs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)}- chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)+ // Buffer the chunk in memory from the global pool for retries+ rw := multipart.NewRW()+ var n int64+ n, err = io.CopyN(rw, in, chunkSize)+ if err != nil && err != io.EOF {+ _ = rw.Close()+ return nil, err+ }skip := int64(0)err = o.fs.pacer.Call(func() (bool, error) {// seek to the start in case this is a retry- if _, err = chunk.Seek(skip, io.SeekStart); err != nil {+ if _, err = rw.Seek(skip, io.SeekStart); err != nil {return false, err}- err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, chunk)+ err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)// after session is started, we retry everythingif err != nil {// Check for incorrect offset error and retry with new offset@@ -2122,10 +2125,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *fwhat := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)if skip < 0 {return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)- } else if skip == chunkSize {+ } else if skip == n {fs.Debugf(o, "%s: chunk received OK - continuing", what)return false, nil- } else if skip > chunkSize {+ } else if skip > n {// This error should never happenreturn false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)}@@ -2142,9 +2145,13 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f}return err != nil, err})+ closeErr := rw.Close()if err != nil {return nil, err}+ if closeErr != nil {+ return nil, closeErr+ }if size >= 0 {// Check for sources which truncate earlyexpected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))