Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

pikpak: fix uploads failing when the server is slow to complete them

Nick Craig-Wood9751d168bcb5
diff --git a/backend/pikpak/helper.go b/backend/pikpak/helper.go
index ee0327553..31ccf7a42 100644
--- a/backend/pikpak/helper.go
+++ b/backend/pikpak/helper.go
@@ -184,7 +184,7 @@ func (f *Fs) patchFile(ctx context.Context, ID string, req *api.File) (info *api
}
// getTask gets api.Task from API for the ID passed
-func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api.Task, err error) {
+func (f *Fs) getTask(ctx context.Context, ID string) (info *api.Task, err error) {
opts := rest.Opts{
Method: "GET",
Path: "/drive/v1/tasks/" + ID,
@@ -192,27 +192,48 @@ func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
- if checkPhase {
- if err == nil && info.Phase != api.PhaseTypeComplete {
- // could be pending right after the task is created
- return true, fmt.Errorf("%s (%s) is still in %s", info.Name, info.Type, info.Phase)
- }
- }
return f.shouldRetry(ctx, resp, err)
})
return
}
-// waitTask waits for async tasks to be completed
+// waitTask waits up to taskTimeout for the async task ID to complete
func (f *Fs) waitTask(ctx context.Context, ID string) (err error) {
- time.Sleep(taskWaitTime)
- if info, err := f.getTask(ctx, ID, true); err != nil {
- if info == nil {
- return fmt.Errorf("can't verify the task is completed: %q", ID)
+ start := time.Now()
+ sleep := taskWaitTime
+ for {
+ if err := sleepCtx(ctx, sleep); err != nil {
+ return err
+ }
+ info, err := f.getTask(ctx, ID)
+ if err != nil {
+ return fmt.Errorf("can't verify the task %q is completed: %w", ID, err)
+ }
+ switch info.Phase {
+ case api.PhaseTypeComplete:
+ fs.Debugf(f, "task %q (%s) completed after %v", ID, info.Type, time.Since(start))
+ return nil
+ case api.PhaseTypeError:
+ return fmt.Errorf("task failed: %#v", info)
}
- return fmt.Errorf("can't verify the task is completed: %#v", info)
+ // Tasks can stay pending for tens of seconds when the server is busy
+ if time.Since(start) > taskTimeout {
+ return fmt.Errorf("can't verify the task is completed after %v: %#v", time.Since(start), info)
+ }
+ sleep = min(2*sleep, taskMaxSleep)
+ }
+}
+
+// sleepCtx sleeps for d or until ctx is cancelled
+func sleepCtx(ctx context.Context, d time.Duration) error {
+ timer := time.NewTimer(d)
+ defer timer.Stop()
+ select {
+ case <-timer.C:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
}
- return
}
// deleteTask remove a task having the specified ID
diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.go
index dd2e1cbe7..754ec0b3b 100644
--- a/backend/pikpak/pikpak.go
+++ b/backend/pikpak/pikpak.go
@@ -72,6 +72,8 @@ const (
minSleep = 100 * time.Millisecond
maxSleep = 2 * time.Second
taskWaitTime = 500 * time.Millisecond
+ taskMaxSleep = 5 * time.Second
+ taskTimeout = 5 * time.Minute
decayConstant = 2 // bigger for slower decay, exponential
rootURL = "https://api-drive.mypikpak.com"