Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

onedrive: re-add --onedrive-upload-cutoff flag

Nick Craig-Woodc37fe733dfd0
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go
index c1a13f654..e425f29b6 100644
--- a/backend/onedrive/onedrive.go
+++ b/backend/onedrive/onedrive.go
@@ -56,6 +56,7 @@ const (
driveTypeSharepoint = "documentLibrary"
defaultChunkSize = 10 * fs.Mebi
chunkSizeMultiple = 320 * fs.Kibi
+ maxSinglePartSize = 4 * fs.Mebi
regionGlobal = "global"
regionUS = "us"
@@ -138,6 +139,21 @@ func init() {
Help: "Azure and Office 365 operated by Vnet Group in China",
},
},
+ }, {
+ Name: "upload_cutoff",
+ Help: `Cutoff for switching to chunked upload.
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+
+This is disabled by default as uploading using single part uploads
+causes rclone to use twice the storage on Onedrive business as when
+rclone sets the modification time after the upload Onedrive creates a
+new version.
+
+See: https://github.com/rclone/rclone/issues/1716
+`,
+ Default: fs.SizeSuffix(-1),
+ Advanced: true,
}, {
Name: "chunk_size",
Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
@@ -746,6 +762,7 @@ Examples:
// Options defines the configuration for this backend
type Options struct {
Region string `config:"region"`
+ UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
DriveID string `config:"drive_id"`
DriveType string `config:"drive_type"`
@@ -1022,6 +1039,13 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
return
}
+func checkUploadCutoff(cs fs.SizeSuffix) error {
+ if cs > maxSinglePartSize {
+ return fmt.Errorf("%v is greater than %v", cs, maxSinglePartSize)
+ }
+ return nil
+}
+
// NewFs constructs an Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -1035,6 +1059,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil {
return nil, fmt.Errorf("onedrive: chunk size: %w", err)
}
+ err = checkUploadCutoff(opt.UploadCutoff)
+ if err != nil {
+ return nil, fmt.Errorf("onedrive: upload cutoff: %w", err)
+ }
if opt.DriveID == "" || opt.DriveType == "" {
return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
@@ -2567,8 +2595,8 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
// This function will set modtime and metadata after uploading, which will create a new version for the remote file
func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {
size := src.Size()
- if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
- return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4 MiB")
+ if size < 0 || size > int64(maxSinglePartSize) {
+ return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and <= %v", maxSinglePartSize)
}
fs.Debugf(o, "Starting singlepart upload")
@@ -2621,9 +2649,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
size := src.Size()
var info *api.Item
- if size > 0 {
+ if size > 0 && size >= int64(o.fs.opt.UploadCutoff) {
info, err = o.uploadMultipart(ctx, in, src, options...)
- } else if size == 0 {
+ } else if size >= 0 {
info, err = o.uploadSinglepart(ctx, in, src, options...)
} else {
return errors.New("unknown-sized upload not supported")