"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pikpak: implement configurable chunk size for multipart upload
wiserainb7624287ac05
diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.goindex 9b6a527cf..4a3fbd3d9 100644--- a/backend/pikpak/pikpak.go+++ b/backend/pikpak/pikpak.go@@ -18,7 +18,6 @@ package pikpak// ------------------------------------------------------------// * List() with options starred-only-// * uploadByResumable() with configurable chunk-size// * user-configurable list chunk// * backend command: untrash, iscached// * api(event,task)@@ -47,6 +46,7 @@ import ("github.com/rclone/rclone/backend/pikpak/api""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/chunksize""github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/config/configstruct"@@ -72,6 +72,8 @@ const (waitTime = 500 * time.MilliseconddecayConstant = 2 // bigger for slower decay, exponentialrootURL = "https://api-drive.mypikpak.com"+ minChunkSize = fs.SizeSuffix(s3manager.MinUploadPartSize)+ defaultUploadConcurrency = s3manager.DefaultUploadConcurrency)// Globals@@ -193,6 +195,42 @@ Fill in for rclone to use a non root folder as its starting point.Help: "Files bigger than this will be cached on disk to calculate hash if required.",Default: fs.SizeSuffix(10 * 1024 * 1024),Advanced: true,+ }, {+ Name: "chunk_size",+ Help: `Chunk size for multipart uploads.++Large files will be uploaded in chunks of this size.++Note that this is stored in memory and there may be up to+"--transfers" * "--pikpak-upload-concurrency" chunks stored at once+in memory.++If you are transferring large files over high-speed links and you have+enough memory, then increasing this will speed up the transfers.++Rclone will automatically increase the chunk size when uploading a+large file of known size to stay below the 10,000 chunks limit.++Increasing the chunk size decreases the accuracy of the progress+statistics displayed with "-P" flag.`,+ Default: minChunkSize,+ Advanced: true,+ }, {+ Name: "upload_concurrency",+ Help: `Concurrency for multipart uploads.++This is the number of chunks of the same file that are uploaded+concurrently for multipart uploads.++Note that chunks are stored in memory and there may be up to+"--transfers" * "--pikpak-upload-concurrency" chunks stored at once+in memory.++If you are uploading small numbers of large files over high-speed links+and these uploads do not fully utilize your bandwidth, then increasing+this may help to speed up the transfers.`,+ Default: defaultUploadConcurrency,+ Advanced: true,}, {Name: config.ConfigEncoding,Help: config.ConfigEncodingHelp,@@ -223,6 +261,8 @@ type Options struct {UseTrash bool `config:"use_trash"`TrashedOnly bool `config:"trashed_only"`HashMemoryThreshold fs.SizeSuffix `config:"hash_memory_limit"`+ ChunkSize fs.SizeSuffix `config:"chunk_size"`+ UploadConcurrency int `config:"upload_concurrency"`Enc encoder.MultiEncoder `config:"encoding"`}@@ -432,6 +472,9 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, errif err := configstruct.Set(m, opt); err != nil {return nil, err}+ if opt.ChunkSize < minChunkSize {+ return nil, fmt.Errorf("chunk size must be at least %s", minChunkSize)+ }root := parsePath(path)@@ -1148,7 +1191,7 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size ireturn}-func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, resumable *api.Resumable) (err error) {+func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable) (err error) {p := resumable.Paramsendpoint := strings.Join(strings.Split(p.Endpoint, ".")[1:], ".") // "mypikpak.com"@@ -1161,19 +1204,21 @@ func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, resumable *apiif err != nil {return}+ partSize := chunksize.Calculator(name, size, s3manager.MaxUploadParts, f.opt.ChunkSize)- uploader := s3manager.NewUploader(sess)+ // Create an uploader with the session and custom options+ uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) {+ u.PartSize = int64(partSize)+ u.Concurrency = f.opt.UploadConcurrency+ })// Upload input parametersuParams := &s3manager.UploadInput{Bucket: &p.Bucket,Key: &p.Key,Body: in,}- // Perform upload with options different than the those in the Uploader.- _, err = uploader.UploadWithContext(ctx, uParams, func(u *s3manager.Uploader) {- // TODO can be user-configurable- u.PartSize = 10 * 1024 * 1024 // 10MB part size- })+ // Perform an upload+ _, err = uploader.UploadWithContext(ctx, uParams)return}@@ -1224,7 +1269,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, sha1Str striif uploadType == api.UploadTypeForm && new.Form != nil {err = f.uploadByForm(ctx, in, req.Name, size, new.Form, options...)} else if uploadType == api.UploadTypeResumable && new.Resumable != nil {- err = f.uploadByResumable(ctx, in, new.Resumable)+ err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable)} else {err = fmt.Errorf("no method available for uploading: %+v", new)}