"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pikpak: fix uploads failing when the server is slow to complete them
Nick Craig-Wood9751d168bcb5
diff --git a/backend/pikpak/helper.go b/backend/pikpak/helper.goindex ee0327553..31ccf7a42 100644--- a/backend/pikpak/helper.go+++ b/backend/pikpak/helper.go@@ -184,7 +184,7 @@ func (f *Fs) patchFile(ctx context.Context, ID string, req *api.File) (info *api}// getTask gets api.Task from API for the ID passed-func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api.Task, err error) {+func (f *Fs) getTask(ctx context.Context, ID string) (info *api.Task, err error) {opts := rest.Opts{Method: "GET",Path: "/drive/v1/tasks/" + ID,@@ -192,27 +192,48 @@ func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *apivar resp *http.Responseerr = f.pacer.Call(func() (bool, error) {resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)- if checkPhase {- if err == nil && info.Phase != api.PhaseTypeComplete {- // could be pending right after the task is created- return true, fmt.Errorf("%s (%s) is still in %s", info.Name, info.Type, info.Phase)- }- }return f.shouldRetry(ctx, resp, err)})return}-// waitTask waits for async tasks to be completed+// waitTask waits up to taskTimeout for the async task ID to completefunc (f *Fs) waitTask(ctx context.Context, ID string) (err error) {- time.Sleep(taskWaitTime)- if info, err := f.getTask(ctx, ID, true); err != nil {- if info == nil {- return fmt.Errorf("can't verify the task is completed: %q", ID)+ start := time.Now()+ sleep := taskWaitTime+ for {+ if err := sleepCtx(ctx, sleep); err != nil {+ return err+ }+ info, err := f.getTask(ctx, ID)+ if err != nil {+ return fmt.Errorf("can't verify the task %q is completed: %w", ID, err)+ }+ switch info.Phase {+ case api.PhaseTypeComplete:+ fs.Debugf(f, "task %q (%s) completed after %v", ID, info.Type, time.Since(start))+ return nil+ case api.PhaseTypeError:+ return fmt.Errorf("task failed: %#v", info)}- return fmt.Errorf("can't verify the task is completed: %#v", info)+ // Tasks can stay pending for tens of seconds when the server is busy+ if time.Since(start) > taskTimeout {+ return fmt.Errorf("can't verify the task is completed after %v: %#v", time.Since(start), info)+ }+ sleep = min(2*sleep, taskMaxSleep)+ }+}++// sleepCtx sleeps for d or until ctx is cancelled+func sleepCtx(ctx context.Context, d time.Duration) error {+ timer := time.NewTimer(d)+ defer timer.Stop()+ select {+ case <-timer.C:+ return nil+ case <-ctx.Done():+ return ctx.Err()}- return}// deleteTask remove a task having the specified IDdiff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.goindex dd2e1cbe7..754ec0b3b 100644--- a/backend/pikpak/pikpak.go+++ b/backend/pikpak/pikpak.go@@ -72,6 +72,8 @@ const (minSleep = 100 * time.MillisecondmaxSleep = 2 * time.SecondtaskWaitTime = 500 * time.Millisecond+ taskMaxSleep = 5 * time.Second+ taskTimeout = 5 * time.MinutedecayConstant = 2 // bigger for slower decay, exponentialrootURL = "https://api-drive.mypikpak.com"