"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pikpak: improve upload reliability and resolve potential file conflicts
wiseraince5024bf3317
diff --git a/backend/pikpak/api/types.go b/backend/pikpak/api/types.goindex 8b2270e2a..6046ee48b 100644--- a/backend/pikpak/api/types.go+++ b/backend/pikpak/api/types.go@@ -251,10 +251,12 @@ type Media struct {// FileParams includes parameters for instant opentype FileParams struct {+ DeviceID string `json:"device_id,omitempty"`Duration int64 `json:"duration,omitempty,string"` // in secondsHeight int `json:"height,omitempty,string"`Platform string `json:"platform,omitempty"` // "Upload"PlatformIcon string `json:"platform_icon,omitempty"`+ TaskID string `json:"task_id"`URL string `json:"url,omitempty"`Width int `json:"width,omitempty,string"`}diff --git a/backend/pikpak/helper.go b/backend/pikpak/helper.goindex 36ae1a03e..6351ef1b4 100644--- a/backend/pikpak/helper.go+++ b/backend/pikpak/helper.go@@ -9,7 +9,9 @@ import ("fmt""io""net/http"+ "net/url""os"+ "strconv""github.com/rclone/rclone/backend/pikpak/api""github.com/rclone/rclone/lib/rest"@@ -141,9 +143,8 @@ func (f *Fs) getFile(ctx context.Context, ID string) (info *api.File, err error)var resp *http.Responseerr = f.pacer.Call(func() (bool, error) {resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)- if err == nil && info.Phase != api.PhaseTypeComplete {- // could be pending right after file is created/uploaded.- return true, errors.New("not PHASE_TYPE_COMPLETE")+ if err == nil && !info.Links.ApplicationOctetStream.Valid() {+ return true, errors.New("no link")}return f.shouldRetry(ctx, resp, err)})@@ -167,6 +168,45 @@ func (f *Fs) patchFile(ctx context.Context, ID string, req *api.File) (info *apireturn}+// getTask gets api.Task from API for the ID passed+func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api.Task, err error) {+ opts := rest.Opts{+ Method: "GET",+ Path: "/drive/v1/tasks/" + ID,+ }+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)+ if checkPhase {+ if err == nil && info.Phase != api.PhaseTypeComplete {+ // could be pending right after file is created/uploaded.+ return true, errors.New(info.Phase)+ }+ }+ return f.shouldRetry(ctx, resp, err)+ })+ return+}++// deleteTask remove a task having the specified ID+func (f *Fs) deleteTask(ctx context.Context, ID string, deleteFiles bool) (err error) {+ params := url.Values{}+ params.Set("delete_files", strconv.FormatBool(deleteFiles))+ params.Set("task_ids", ID)+ opts := rest.Opts{+ Method: "DELETE",+ Path: "/drive/v1/tasks",+ Parameters: params,+ NoResponse: true,+ }+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.rst.CallJSON(ctx, &opts, nil, nil)+ return f.shouldRetry(ctx, resp, err)+ })+ return+}+// getAbout gets drive#quota information from serverfunc (f *Fs) getAbout(ctx context.Context) (info *api.About, err error) {opts := rest.Opts{diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.goindex 008180356..9b6a527cf 100644--- a/backend/pikpak/pikpak.go+++ b/backend/pikpak/pikpak.go@@ -53,6 +53,7 @@ import ("github.com/rclone/rclone/fs/config/obscure""github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/lib/atexit""github.com/rclone/rclone/lib/dircache""github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/oauthutil"@@ -66,8 +67,9 @@ import (const (rcloneClientID = "YNxT9w7GMdWvEOKa"rcloneEncryptedClientSecret = "aqrmB6M1YJ1DWCBxVxFSjFo7wzWEky494YMmkqgAl1do1WKOe2E"- minSleep = 10 * time.Millisecond+ minSleep = 100 * time.MillisecondmaxSleep = 2 * time.Second+ waitTime = 500 * time.MilliseconddecayConstant = 2 // bigger for slower decay, exponentialrootURL = "https://api-drive.mypikpak.com")@@ -1175,12 +1177,13 @@ func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, resumable *apireturn}-func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, sha1Str string, size int64, options ...fs.OpenOption) (*api.File, error) {+func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, sha1Str string, size int64, options ...fs.OpenOption) (info *api.File, err error) {// determine upload typeuploadType := api.UploadTypeResumable- if size >= 0 && size < int64(5*fs.Mebi) {- uploadType = api.UploadTypeForm- }+ // if size >= 0 && size < int64(5*fs.Mebi) {+ // uploadType = api.UploadTypeForm+ // }+ // stop using uploadByForm() cause it is not as reliable as uploadByResumable() for a large number of small files// request upload ticket to APIreq := api.RequestNewFile{@@ -1195,29 +1198,46 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, sha1Str striif uploadType == api.UploadTypeResumable {req.Resumable = map[string]string{"provider": "PROVIDER_ALIYUN"}}- newfile, err := f.requestNewFile(ctx, &req)+ new, err := f.requestNewFile(ctx, &req)if err != nil {return nil, fmt.Errorf("failed to create a new file: %w", err)}- if newfile.File == nil {- return nil, fmt.Errorf("invalid response: %+v", newfile)- } else if newfile.File.Phase == api.PhaseTypeComplete {+ if new.File == nil {+ return nil, fmt.Errorf("invalid response: %+v", new)+ } else if new.File.Phase == api.PhaseTypeComplete {// early return; in case of zero-byte objects- return newfile.File, nil+ return new.File, nil}- if uploadType == api.UploadTypeForm && newfile.Form != nil {- err = f.uploadByForm(ctx, in, req.Name, size, newfile.Form, options...)- } else if uploadType == api.UploadTypeResumable && newfile.Resumable != nil {- err = f.uploadByResumable(ctx, in, newfile.Resumable)+ defer atexit.OnError(&err, func() {+ fs.Debugf(leaf, "canceling upload: %v", err)+ if cancelErr := f.deleteObjects(ctx, []string{new.File.ID}, false); cancelErr != nil {+ fs.Logf(leaf, "failed to cancel upload: %v", cancelErr)+ }+ if cancelErr := f.deleteTask(ctx, new.Task.ID, false); cancelErr != nil {+ fs.Logf(leaf, "failed to cancel upload: %v", cancelErr)+ }+ fs.Debugf(leaf, "waiting %v for the cancellation to be effective", waitTime)+ time.Sleep(waitTime)+ })()++ if uploadType == api.UploadTypeForm && new.Form != nil {+ err = f.uploadByForm(ctx, in, req.Name, size, new.Form, options...)+ } else if uploadType == api.UploadTypeResumable && new.Resumable != nil {+ err = f.uploadByResumable(ctx, in, new.Resumable)} else {- return nil, fmt.Errorf("unable to proceed upload: %+v", newfile)+ err = fmt.Errorf("no method available for uploading: %+v", new)}if err != nil {return nil, fmt.Errorf("failed to upload: %w", err)}- return newfile.File, nil+ fs.Debugf(leaf, "sleeping for %v before checking upload status", waitTime)+ time.Sleep(waitTime)+ if _, err = f.getTask(ctx, new.Task.ID, true); err != nil {+ return nil, fmt.Errorf("unable to complete the upload: %w", err)+ }+ return new.File, nil}// Put the object@@ -1470,22 +1490,11 @@ func (o *Object) setMetaDataWithLink(ctx context.Context) error {return nil}- // fetch download link with retry scheme- // 1 initial attempt and 2 retries are reasonable based on empirical analysis- retries := 2- for i := 1; i <= retries+1; i++ {- info, err := o.fs.getFile(ctx, o.id)- if err != nil {- return fmt.Errorf("can't fetch download link: %w", err)- }- if err = o.setMetaData(info); err == nil && o.link.Valid() {- return nil- }- if i <= retries {- time.Sleep(time.Duration(200*i) * time.Millisecond)- }+ info, err := o.fs.getFile(ctx, o.id)+ if err != nil {+ return err}- return errors.New("can't download - no link to download")+ return o.setMetaData(info)}// readMetaData gets the metadata if it hasn't already been fetched@@ -1619,14 +1628,14 @@ func (o *Object) open(ctx context.Context, url string, options ...fs.OpenOption)// Open an object for readfunc (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {if o.id == "" {- return nil, errors.New("can't download - no id")+ return nil, errors.New("can't download: no id")}if o.size == 0 {// zero-byte objects may have no download linkreturn io.NopCloser(bytes.NewBuffer([]byte(nil))), nil}if err = o.setMetaDataWithLink(ctx); err != nil {- return nil, err+ return nil, fmt.Errorf("can't download: %w", err)}return o.open(ctx, o.link.URL, options...)}