"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
opendrive: reuse upload chunk buffers via the global memory pool
Nick Craig-Wood6613d4ec637b
diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.goindex 86e3a0285..e48374ff3 100644--- a/backend/opendrive/opendrive.go+++ b/backend/opendrive/opendrive.go@@ -23,8 +23,8 @@ import ("github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/lib/dircache""github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/pacer"- "github.com/rclone/rclone/lib/readers""github.com/rclone/rclone/lib/rest")@@ -1047,7 +1047,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op// resp.Body.Close()// fs.Debugf(nil, "PostOpen: %#v", openResponse)- buf := make([]byte, o.fs.opt.ChunkSize)chunkOffset := int64(0)remainingBytes := sizechunkCounter := 0@@ -1057,17 +1056,27 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opremainingBytes -= currentChunkSizefs.Debugf(o, "Uploading chunk %d, size=%d, remain=%d", chunkCounter, currentChunkSize, remainingBytes)- chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, currentChunkSize)+ // Buffer the chunk in memory from the global pool so reads are+ // repeatable for retries+ rw := multipart.NewRW()+ _, err = io.CopyN(rw, in, currentChunkSize)+ if err != nil {+ _ = rw.Close()+ if err == io.EOF {+ err = fmt.Errorf("short read of chunk %d: %w", chunkCounter, io.ErrUnexpectedEOF)+ }+ return fmt.Errorf("failed to create file: %w", err)+ }var reply uploadFileChunkReplyerr = o.fs.pacer.Call(func() (bool, error) {// seek to the start in case this is a retry- if _, err = chunk.Seek(0, io.SeekStart); err != nil {+ if _, err = rw.Seek(0, io.SeekStart); err != nil {return false, err}opts := rest.Opts{Method: "POST",Path: "/upload/upload_file_chunk.json",- Body: chunk,+ Body: rw,MultipartParams: url.Values{"session_id": []string{o.fs.session.SessionID},"file_id": []string{o.id},@@ -1082,9 +1091,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opresp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)return o.fs.shouldRetry(ctx, resp, err)})+ closeErr := rw.Close()if err != nil {return fmt.Errorf("failed to create file: %w", err)}+ if closeErr != nil {+ return fmt.Errorf("failed to create file: %w", closeErr)+ }if reply.TotalWritten != currentChunkSize {return fmt.Errorf("failed to create file: incomplete write of %d/%d bytes", reply.TotalWritten, currentChunkSize)}