"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
putio: reuse upload chunk buffers via the global memory pool
Nick Craig-Woodcdabcc7cc42d
diff --git a/backend/putio/fs.go b/backend/putio/fs.goindex a0570c971..158520d88 100644--- a/backend/putio/fs.go+++ b/backend/putio/fs.go@@ -21,6 +21,7 @@ import ("github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/lib/dircache"+ "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/oauthutil""github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/random"@@ -330,17 +331,31 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.}var clientOffset int64var offsetMismatch bool- buf := make([]byte, defaultChunkSize)for clientOffset < size {chunkSize := min(size-clientOffset, int64(defaultChunkSize))- chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)chunkStart := clientOffsetreqSize := chunkSizetransferOffset := clientOffsetfs.Debugf(f, "chunkStart: %d, reqSize: %d", chunkStart, reqSize)+ // Buffer the chunk in memory from the global pool so reads are+ // repeatable for retries+ rw := multipart.NewRW()+ _, err = io.CopyN(rw, in, chunkSize)+ if err != nil {+ _ = rw.Close()+ if err == io.EOF {+ err = fmt.Errorf("short read of chunk at %d: %w", chunkStart, io.ErrUnexpectedEOF)+ }+ return+ }+// Transfer the chunkerr = f.pacer.Call(func() (bool, error) {+ _, err = rw.Seek(0, io.SeekStart)+ if err != nil {+ return false, err+ }if offsetMismatch {// Get file offset and seek to the positionoffset, err := f.getServerOffset(ctx, location)@@ -349,7 +364,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.}sentBytes := offset - chunkStartfs.Debugf(f, "sentBytes: %d", sentBytes)- _, err = chunk.Seek(sentBytes, io.SeekStart)+ _, err = rw.Seek(sentBytes, io.SeekStart)if err != nil {return shouldRetry(ctx, err)}@@ -359,7 +374,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.}fs.Debugf(f, "Sending chunk. transferOffset: %d length: %d", transferOffset, reqSize)var serverOffset int64- serverOffset, fileID, err = f.transferChunk(ctx, location, transferOffset, chunk, reqSize)+ serverOffset, fileID, err = f.transferChunk(ctx, location, transferOffset, rw, reqSize)if cerr, ok := err.(*statusCodeError); ok && cerr.response.StatusCode == 409 {offsetMismatch = truereturn true, err@@ -370,9 +385,13 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.}return shouldRetry(ctx, err)})+ closeErr := rw.Close()if err != nil {return}+ if closeErr != nil {+ return 0, closeErr+ }clientOffset += chunkSize}@@ -436,11 +455,19 @@ func (f *Fs) makeUploadHeadRequest(ctx context.Context, location string) (*http.return req, nil}+// makeUploadPatchRequest makes the tus PATCH request sending length bytes+// of in at offset.+//+// The body is wrapped in readers.NoCloser so the transport can't upgrade it+// to an io.Closer and close it after each attempt — the chunk buffer is+// pool-backed, so an early close would free its pages and a retry would then+// read a dead buffer. The upload loop owns the buffer's lifetime.func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.Reader, offset, length int64) (*http.Request, error) {- req, err := http.NewRequestWithContext(ctx, "PATCH", location, in)+ req, err := http.NewRequestWithContext(ctx, "PATCH", location, readers.NoCloser(in))if err != nil {return nil, err}+ req.ContentLength = lengthreq.Header.Set("tus-resumable", "1.0.0")req.Header.Set("upload-offset", strconv.FormatInt(offset, 10))req.Header.Set("content-length", strconv.FormatInt(length, 10))