Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

hidrive: reuse chunked upload buffers via the global memory pool

Nick Craig-Woode8034e654856
diff --git a/backend/hidrive/helpers.go b/backend/hidrive/helpers.go
index c64d53171..3e7ac374d 100644
--- a/backend/hidrive/helpers.go
+++ b/backend/hidrive/helpers.go
@@ -24,6 +24,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/ranges"
"github.com/rclone/rclone/lib/readers"
@@ -637,11 +638,18 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
}
// Read a chunk of data.
- chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
- if bytesRead < chunkSize {
+ var (
+ chunkReader *pool.RW
+ bytesRead int64
+ )
+ chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize))
+ if bytesRead < int64(chunkSize) {
startMoreTransfers = false
}
if readErr != nil || bytesRead <= 0 {
+ if chunkReader != nil {
+ _ = chunkReader.Close()
+ }
break
}
@@ -651,11 +659,12 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
// After this upload is done,
// signal that another transfer can be started.
defer transferSemaphore.Release(1)
- uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
+ defer func() { _ = chunkReader.Close() }()
+ uploadErr := f.patchFile(gCtx, path, chunkReader, chunkOffset, zeroTime)
if uploadErr == nil {
// Remember successfully written chunks.
okChunksMu.Lock()
- okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
+ okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: bytesRead})
okChunksMu.Unlock()
fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
} else {
@@ -699,10 +708,10 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
// starting at the given offset.
//
// Replaces the file contents starting from the given byte offset
-// with the content of the io.ReadSeeker.
+// with the content of the buffer.
// If the offset is beyond the file end, the file is extended up to the offset.
// The maximum size of the update is limited by MaximumUploadBytes.
-// The io.ReadSeeker should be resettable by seeking to its start.
+// The caller remains responsible for closing the buffer.
// If modTime is not the zero time instant,
// it will be set as the file's modification time after the operation.
//
@@ -714,7 +723,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
// usually expand the holes into regions of 0-bytes.
//
// This returns fs.ErrorObjectNotFound if the object is not found.
-func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
+func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offset uint64, modTime time.Time) error {
parameters := api.NewQueryParameters()
parameters.SetPath(path)
parameters.Set("offset", strconv.FormatUint(offset, 10))
@@ -726,13 +735,15 @@ func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker,
}
}
+ contentLength := content.Size()
opts := rest.Opts{
- Method: "PATCH",
- Path: "/file",
- Body: content,
- ContentType: "application/octet-stream",
- Parameters: parameters.Values,
- NoResponse: true,
+ Method: "PATCH",
+ Path: "/file",
+ Body: content,
+ ContentType: "application/octet-stream",
+ ContentLength: &contentLength,
+ Parameters: parameters.Values,
+ NoResponse: true,
}
var resp *http.Response
@@ -876,25 +887,28 @@ func cachedReader(reader io.Reader) io.ReadSeeker {
return readers.NewRepeatableReader(reader)
}
-// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
-// Returns a new io.Reader (chunkReader) for that chunk
-// and the number of bytes that have been read from reader.
-func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
- // Unwrap any accounting from the input if present.
- reader, wrap := accounting.UnWrap(reader)
-
- // Read a chunk of data.
- buffer := make([]byte, length)
- bytesRead, err = io.ReadFull(reader, buffer)
- if err == io.EOF || err == io.ErrUnexpectedEOF {
+// readerForChunk reads up to length bytes from reader into a buffer
+// from the global memory pool and returns it
+// along with the number of bytes that have been read from reader.
+// Reaching the end of reader early is not an error.
+//
+// Any accounting on reader is applied when the buffer is read instead,
+// so the upload is what gets accounted.
+// The caller must Close the buffer when done with it.
+func readerForChunk(reader io.Reader, length int64) (chunkReader *pool.RW, bytesRead int64, err error) {
+ reader, acc := unwrapAccounting(reader)
+ chunkReader = multipart.NewRW()
+ if acc != nil {
+ chunkReader.SetAccounting(acc.AccountRead)
+ }
+
+ bytesRead, err = io.CopyN(chunkReader, reader, length)
+ if err == io.EOF {
err = nil
}
if err != nil {
+ _ = chunkReader.Close()
return nil, bytesRead, err
}
- // Truncate unused capacity.
- buffer = buffer[:bytesRead]
-
- // Use wrap to put any accounting back for chunkReader.
- return wrap(bytes.NewReader(buffer)), bytesRead, nil
+ return chunkReader, bytesRead, nil
}
diff --git a/backend/hidrive/hidrive.go b/backend/hidrive/hidrive.go
index a84dbaa09..90f2652dd 100644
--- a/backend/hidrive/hidrive.go
+++ b/backend/hidrive/hidrive.go
@@ -28,7 +28,6 @@ import (
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
@@ -522,17 +521,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
if size := src.Size(); size >= 0 && size < prefixSize {
prefixSize = size
}
- unwrapped, acc := unwrapAccounting(in)
- cutoffReader := multipart.NewRW()
- if acc != nil {
- cutoffReader.SetAccounting(acc.AccountRead)
- }
- bytesRead, err := io.CopyN(cutoffReader, unwrapped, prefixSize)
- if err == io.EOF {
- err = nil
- }
+ cutoffReader, bytesRead, err := readerForChunk(in, prefixSize)
if err != nil {
- _ = cutoffReader.Close()
return nil, err
}