"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
filelu: reuse multipart upload buffers via the global pool and retry failed parts
Nick Craig-Woodf5bdceab49c4
diff --git a/backend/filelu/filelu_file_uploader.go b/backend/filelu/filelu_file_uploader.goindex d868e85c1..12db35565 100644--- a/backend/filelu/filelu_file_uploader.go+++ b/backend/filelu/filelu_file_uploader.go@@ -1,7 +1,6 @@package fileluimport (- "bytes""context""encoding/json""errors"@@ -11,9 +10,11 @@ import ("net/http""net/url""path"+ "strconv""strings""github.com/rclone/rclone/fs"+ rclonemultipart "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/rest")@@ -47,39 +48,31 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) eserver := initResp.Result.ServerobjectPath := initResp.Result.ObjectPath- chunkSize := int(f.opt.ChunkSize)- buf := make([]byte, 0, chunkSize)- tmp := make([]byte, 1024*1024)- partNo := 1-- for {- n, errRead := in.Read(tmp)+ chunkSize := int64(f.opt.ChunkSize)+ if chunkSize <= 0 {+ return fmt.Errorf("multipart upload: chunk_size must be positive: %v", f.opt.ChunkSize)+ }+ for partNo := 1; ; partNo++ {+ // Buffer the part in memory from the global pool so it can be+ // re-sent on retry+ rw := rclonemultipart.NewRW()+ n, err := io.CopyN(rw, in, chunkSize)+ if err != nil && err != io.EOF {+ _ = rw.Close()+ return fmt.Errorf("read failed: %w", err)+ }if n > 0 {- buf = append(buf, tmp[:n]...)-- // If buffer reached chunkSize, upload a full part- if len(buf) >= chunkSize {- err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))- if err != nil {- return fmt.Errorf("upload part %d failed: %w", partNo, err)- }- partNo++- buf = buf[:0]+ uploadErr := f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, rw, n)+ if uploadErr != nil {+ _ = rw.Close()+ return fmt.Errorf("upload part %d failed: %w", partNo, uploadErr)}}-- if errRead == io.EOF {- break- }- if errRead != nil {- return fmt.Errorf("read failed: %w", errRead)+ if closeErr := rw.Close(); closeErr != nil {+ return closeErr}- }-- if len(buf) > 0 {- err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))- if err != nil {- return fmt.Errorf("upload part %d failed: %w", partNo, err)+ if err == io.EOF {+ break}}@@ -91,31 +84,40 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) ereturn nil}-// uploadPart sends a single multipart chunk to the upload server.-func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {- url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)-- req, err := http.NewRequestWithContext(ctx, "PUT", url, r)- if err != nil {- return err- }-- req.Header.Set("X-RC-Upload-Id", uploadID)- req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))- req.Header.Set("X-Sess-ID", sessID)- req.Header.Set("X-Object-Path", objectPath)-- resp, err := f.client.Do(req)- if err != nil {- return err- }- defer func() { _ = resp.Body.Close() }()-- if resp.StatusCode != 200 {- return fmt.Errorf("uploadPart failed: %s", resp.Status)+// uploadPart sends a single multipart chunk of size bytes to the upload server.+func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.ReadSeeker, size int64) error {+ opts := rest.Opts{+ Method: "PUT",+ RootURL: server,+ Parameters: url.Values{+ "partNumber": {strconv.Itoa(partNo)},+ "uploadId": {uploadID},+ },+ Body: r,+ ContentLength: &size,+ ExtraHeaders: map[string]string{+ "X-RC-Upload-Id": uploadID,+ "X-RC-Part-No": strconv.Itoa(partNo),+ "X-Sess-ID": sessID,+ "X-Object-Path": objectPath,+ },+ NoResponse: true,}-- return nil+ return f.pacer.Call(func() (bool, error) {+ // rewind the pooled buffer so each attempt sends the whole chunk+ _, err := r.Seek(0, io.SeekStart)+ if err != nil {+ return false, err+ }+ resp, err := f.srv.Call(ctx, &opts)+ if err != nil {+ if resp != nil {+ return shouldRetryHTTP(resp.StatusCode), fmt.Errorf("uploadPart failed: %w", err)+ }+ return shouldRetry(err), err+ }+ return false, nil+ })}// uploadFile uploads a file to FileLu