Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

hidrive: fix truncated file left behind when an upload hits --max-transfer

Nick Craig-Wood9f0b40c7d2e4
diff --git a/backend/hidrive/helpers.go b/backend/hidrive/helpers.go
index b0525d647..f25e38980 100644
--- a/backend/hidrive/helpers.go
+++ b/backend/hidrive/helpers.go
@@ -642,7 +642,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
chunkReader *pool.RW
bytesRead int64
)
- chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize))
+ chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize), true)
if bytesRead < int64(chunkSize) {
startMoreTransfers = false
}
@@ -869,14 +869,23 @@ func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Ac
// along with the number of bytes that have been read from reader.
// Reaching the end of reader early is not an error.
//
-// Any accounting on reader is applied when the buffer is read instead,
-// so the upload is what gets accounted.
+// If accountUpload is set, any accounting on reader is applied
+// when the buffer is read instead, so the upload is what gets accounted.
+//
+// This must not be set for a buffer which creates or replaces a file
+// in a single request. Accounting can fail part way through the upload,
+// for example when --max-transfer is reached, and HiDrive keeps
+// what it has received of an aborted request as the file.
+//
// The caller must Close the buffer when done with it.
-func readerForChunk(reader io.Reader, length int64) (chunkReader *pool.RW, bytesRead int64, err error) {
- reader, acc := unwrapAccounting(reader)
+func readerForChunk(reader io.Reader, length int64, accountUpload bool) (chunkReader *pool.RW, bytesRead int64, err error) {
chunkReader = multipart.NewRW()
- if acc != nil {
- chunkReader.SetAccounting(acc.AccountRead)
+ if accountUpload {
+ var acc *accounting.Account
+ reader, acc = unwrapAccounting(reader)
+ if acc != nil {
+ chunkReader.SetAccounting(acc.AccountRead)
+ }
}
bytesRead, err = io.CopyN(chunkReader, reader, length)
diff --git a/backend/hidrive/hidrive.go b/backend/hidrive/hidrive.go
index 8b4e1d9a9..0c652d5bb 100644
--- a/backend/hidrive/hidrive.go
+++ b/backend/hidrive/hidrive.go
@@ -522,7 +522,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
if size := src.Size(); size >= 0 && size < prefixSize {
prefixSize = size
}
- cutoffReader, bytesRead, err := readerForChunk(in, prefixSize)
+ cutoffReader, bytesRead, err := readerForChunk(in, prefixSize, false)
if err != nil {
return nil, err
}
@@ -980,7 +980,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
} else {
var content *pool.RW
- content, _, err = readerForChunk(in, src.Size())
+ content, _, err = readerForChunk(in, src.Size(), false)
if err == nil {
info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
_ = content.Close()