"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
hidrive: fix truncated file left behind when an upload hits --max-transfer
Nick Craig-Wood9f0b40c7d2e4
diff --git a/backend/hidrive/helpers.go b/backend/hidrive/helpers.goindex b0525d647..f25e38980 100644--- a/backend/hidrive/helpers.go+++ b/backend/hidrive/helpers.go@@ -642,7 +642,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.ReadchunkReader *pool.RWbytesRead int64)- chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize))+ chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize), true)if bytesRead < int64(chunkSize) {startMoreTransfers = false}@@ -869,14 +869,23 @@ func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Ac// along with the number of bytes that have been read from reader.// Reaching the end of reader early is not an error.//-// Any accounting on reader is applied when the buffer is read instead,-// so the upload is what gets accounted.+// If accountUpload is set, any accounting on reader is applied+// when the buffer is read instead, so the upload is what gets accounted.+//+// This must not be set for a buffer which creates or replaces a file+// in a single request. Accounting can fail part way through the upload,+// for example when --max-transfer is reached, and HiDrive keeps+// what it has received of an aborted request as the file.+//// The caller must Close the buffer when done with it.-func readerForChunk(reader io.Reader, length int64) (chunkReader *pool.RW, bytesRead int64, err error) {- reader, acc := unwrapAccounting(reader)+func readerForChunk(reader io.Reader, length int64, accountUpload bool) (chunkReader *pool.RW, bytesRead int64, err error) {chunkReader = multipart.NewRW()- if acc != nil {- chunkReader.SetAccounting(acc.AccountRead)+ if accountUpload {+ var acc *accounting.Account+ reader, acc = unwrapAccounting(reader)+ if acc != nil {+ chunkReader.SetAccounting(acc.AccountRead)+ }}bytesRead, err = io.CopyN(chunkReader, reader, length)diff --git a/backend/hidrive/hidrive.go b/backend/hidrive/hidrive.goindex 8b4e1d9a9..0c652d5bb 100644--- a/backend/hidrive/hidrive.go+++ b/backend/hidrive/hidrive.go@@ -522,7 +522,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,if size := src.Size(); size >= 0 && size < prefixSize {prefixSize = size}- cutoffReader, bytesRead, err := readerForChunk(in, prefixSize)+ cutoffReader, bytesRead, err := readerForChunk(in, prefixSize, false)if err != nil {return nil, err}@@ -980,7 +980,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opinfo, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)} else {var content *pool.RW- content, _, err = readerForChunk(in, src.Size())+ content, _, err = readerForChunk(in, src.Size(), false)if err == nil {info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)_ = content.Close()