"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
onedrive: re-add --onedrive-upload-cutoff flag
Nick Craig-Woodc37fe733dfd0
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.goindex c1a13f654..e425f29b6 100644--- a/backend/onedrive/onedrive.go+++ b/backend/onedrive/onedrive.go@@ -56,6 +56,7 @@ const (driveTypeSharepoint = "documentLibrary"defaultChunkSize = 10 * fs.MebichunkSizeMultiple = 320 * fs.Kibi+ maxSinglePartSize = 4 * fs.MebiregionGlobal = "global"regionUS = "us"@@ -138,6 +139,21 @@ func init() {Help: "Azure and Office 365 operated by Vnet Group in China",},},+ }, {+ Name: "upload_cutoff",+ Help: `Cutoff for switching to chunked upload.++Any files larger than this will be uploaded in chunks of chunk_size.++This is disabled by default as uploading using single part uploads+causes rclone to use twice the storage on Onedrive business as when+rclone sets the modification time after the upload Onedrive creates a+new version.++See: https://github.com/rclone/rclone/issues/1716+`,+ Default: fs.SizeSuffix(-1),+ Advanced: true,}, {Name: "chunk_size",Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).@@ -746,6 +762,7 @@ Examples:// Options defines the configuration for this backendtype Options struct {Region string `config:"region"`+ UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`ChunkSize fs.SizeSuffix `config:"chunk_size"`DriveID string `config:"drive_id"`DriveType string `config:"drive_type"`@@ -1022,6 +1039,13 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)return}+func checkUploadCutoff(cs fs.SizeSuffix) error {+ if cs > maxSinglePartSize {+ return fmt.Errorf("%v is greater than %v", cs, maxSinglePartSize)+ }+ return nil+}+// NewFs constructs an Fs from the path, container:pathfunc NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {// Parse config into Options struct@@ -1035,6 +1059,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eif err != nil {return nil, fmt.Errorf("onedrive: chunk size: %w", err)}+ err = checkUploadCutoff(opt.UploadCutoff)+ if err != nil {+ return nil, fmt.Errorf("onedrive: upload cutoff: %w", err)+ }if opt.DriveID == "" || opt.DriveType == "" {return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")@@ -2567,8 +2595,8 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec// This function will set modtime and metadata after uploading, which will create a new version for the remote filefunc (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {size := src.Size()- if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {- return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4 MiB")+ if size < 0 || size > int64(maxSinglePartSize) {+ return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and <= %v", maxSinglePartSize)}fs.Debugf(o, "Starting singlepart upload")@@ -2621,9 +2649,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opsize := src.Size()var info *api.Item- if size > 0 {+ if size > 0 && size >= int64(o.fs.opt.UploadCutoff) {info, err = o.uploadMultipart(ctx, in, src, options...)- } else if size == 0 {+ } else if size >= 0 {info, err = o.uploadSinglepart(ctx, in, src, options...)} else {return errors.New("unknown-sized upload not supported")