"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
box: reuse multipart upload part buffers via the global pool
Nick Craig-Woodfb8783732d6b
diff --git a/backend/box/upload.go b/backend/box/upload.goindex a9b3a9f9e..b79af2394 100644--- a/backend/box/upload.go+++ b/backend/box/upload.go@@ -3,7 +3,6 @@package boximport (- "bytes""context""crypto/sha1""encoding/base64"@@ -20,6 +19,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/lib/atexit"+ "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/rest")@@ -55,9 +55,15 @@ func sha1Digest(digest []byte) string {}// uploadPart uploads a part in an upload session-func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {- chunkSize := int64(len(chunk))- sha1sum := sha1.Sum(chunk)+//+// chunk must contain exactly the part's data and is read from the start+// on each attempt.+func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {+ hasher := sha1.New()+ chunkSize, err := io.Copy(hasher, chunk)+ if err != nil {+ return nil, err+ }opts := rest.Opts{Method: "PUT",Path: "/files/upload_sessions/" + SessionID,@@ -67,12 +73,16 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),Options: options,ExtraHeaders: map[string]string{- "Digest": sha1Digest(sha1sum[:]),+ "Digest": sha1Digest(hasher.Sum(nil)),},}var resp *http.Responseerr = o.fs.pacer.Call(func() (bool, error) {- opts.Body = wrap(bytes.NewReader(chunk))+ _, err = chunk.Seek(0, io.SeekStart)+ if err != nil {+ return false, err+ }+ opts.Body = wrap(chunk)resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)return shouldRetry(ctx, resp, err)})@@ -213,27 +223,25 @@ outer:reqSize := min(remaining, chunkSize)- // Make a block of memory- buf := make([]byte, reqSize)-- // Read the chunk- _, err = io.ReadFull(in, buf)+ // Read the chunk into memory from the global pool, hashing+ // it for the whole file hash (must be done sequentially)+ rw := multipart.NewRW().Reserve(reqSize)+ _, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)if err != nil {+ _ = rw.Close()err = fmt.Errorf("multipart upload failed to read source: %w", err)break outer}- // Make the global hash (must be done sequentially)- _, _ = hash.Write(buf)-// Transfer the chunkwg.Add(1)o.fs.uploadToken.Get()go func(part int, position int64) {defer wg.Done()defer o.fs.uploadToken.Put()+ defer func() { _ = rw.Close() }()fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))- partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)+ partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)if err != nil {err = fmt.Errorf("multipart upload failed to upload part: %w", err)select {