"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
shade: retry server errors instead of failing the transfer
Nick Craig-Wooda4972de505af
diff --git a/backend/shade/shade.go b/backend/shade/shade.goindex dc8d1c64e..eacbdc37b 100644--- a/backend/shade/shade.go+++ b/backend/shade/shade.go@@ -22,6 +22,7 @@ import ("github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/object"@@ -42,6 +43,25 @@ const (maxUploadParts = 10000 // maximum allowed number of parts in a multipart upload)+// retryErrorCodes is a slice of error codes that we will retry+var retryErrorCodes = []int{+ 429, // Too Many Requests.+ 500, // Internal Server Error+ 502, // Bad Gateway+ 503, // Service Unavailable+ 504, // Gateway Timeout+ 509, // Bandwidth Limit Exceeded+}++// shouldRetry returns a boolean as to whether this resp and err+// deserve to be retried. It returns the err as a convenience+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {+ if fserrors.ContextError(ctx, &err) {+ return false, err+ }+ return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err+}+// Register with Fsfunc init() {fs.Register(&fs.RegInfo{@@ -126,14 +146,14 @@ func (f *Fs) refreshJWTToken(ctx context.Context) (string, error) {res, err := f.apiSrv.Call(ctx, &opts)if err != nil {fs.Debugf(f, "Token request failed: %v", err)- return false, err+ return shouldRetry(ctx, res, err)}defer fs.CheckClose(res.Body, &err)if res.StatusCode != http.StatusOK {fs.Debugf(f, "Token request failed with code: %d", res.StatusCode)- return res.StatusCode == http.StatusTooManyRequests, fmt.Errorf("failed to get ShadeFS token, status: %d", res.StatusCode)+ return fserrors.ShouldRetryHTTP(res, retryErrorCodes), fmt.Errorf("failed to get ShadeFS token, status: %d", res.StatusCode)}// Read token directly as plain text@@ -202,10 +222,7 @@ func (f *Fs) callAPI(ctx context.Context, method, path string, response any) (*h} else {res, err = f.srv.Call(ctx, &opts)}- if err != nil {- return res != nil && res.StatusCode == http.StatusTooManyRequests, err- }- return false, nil+ return shouldRetry(ctx, res, err)})return res, err}@@ -334,11 +351,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,err = o.fs.pacer.Call(func() (bool, error) {resp, err := f.srv.Call(ctx, &opts)- if err != nil && resp.StatusCode == http.StatusBadRequest {+ if err != nil && resp != nil && resp.StatusCode == http.StatusBadRequest {fs.Debugf(f, "Bad token from server: %v", token)}- return resp != nil && resp.StatusCode == http.StatusTooManyRequests, err+ return shouldRetry(ctx, resp, err)})if err != nil {return nil, err@@ -860,9 +877,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo}res, err = client.Do(req)if err != nil {- return false, err+ return shouldRetry(ctx, nil, err)}- return res.StatusCode == http.StatusTooManyRequests, nil+ return fserrors.ShouldRetryHTTP(res, retryErrorCodes), nil})if err != nil {@@ -899,12 +916,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloerr = o.fs.pacer.Call(func() (bool, error) {downloadRes, err = client.Call(ctx, &opts)if err != nil {- return false, err+ return shouldRetry(ctx, downloadRes, err)}if downloadRes == nil {return false, fmt.Errorf("failed to fetch presigned URL")}- return downloadRes.StatusCode == http.StatusTooManyRequests, nil+ return fserrors.ShouldRetryHTTP(downloadRes, retryErrorCodes), nil})if err != nil {diff --git a/backend/shade/upload.go b/backend/shade/upload.goindex 0419f42d1..89f62bac2 100644--- a/backend/shade/upload.go+++ b/backend/shade/upload.go@@ -17,6 +17,7 @@ import ("github.com/rclone/rclone/backend/shade/api""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/chunksize"+ "github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/rest")@@ -125,10 +126,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInerr = o.fs.pacer.Call(func() (bool, error) {res, err := o.fs.srv.CallJSON(ctx, &opts, reqBody, &initResp)- if err != nil {- return res != nil && res.StatusCode == http.StatusTooManyRequests, err- }- return false, nil+ return shouldRetry(ctx, res, err)})if err != nil {@@ -183,10 +181,7 @@ func (s *shadeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, readerr = s.f.pacer.Call(func() (bool, error) {res, err := s.f.srv.CallJSON(ctx, &partOpts, nil, &partURL)- if err != nil {- return res != nil && res.StatusCode == http.StatusTooManyRequests, err- }- return false, nil+ return shouldRetry(ctx, res, err)})if err != nil {@@ -195,7 +190,6 @@ func (s *shadeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, readopts := rest.Opts{Method: "PUT",RootURL: partURL.URL,- Body: &chunk,ContentType: "",ContentLength: &n,}@@ -208,11 +202,10 @@ func (s *shadeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, read}err = s.f.pacer.Call(func() (bool, error) {+ // Use a fresh reader for each attempt so retries resend the whole chunk+ opts.Body = bytes.NewReader(chunk.Bytes())uploadRes, err = s.f.srv.Call(ctx, &opts)- if err != nil {- return uploadRes != nil && uploadRes.StatusCode == http.StatusTooManyRequests, err- }- return false, nil+ return shouldRetry(ctx, uploadRes, err)})if err != nil {@@ -276,17 +269,13 @@ func (s *shadeChunkWriter) Close(ctx context.Context) error {err = s.f.pacer.Call(func() (bool, error) {res, err := s.f.srv.CallJSON(ctx, &completeOpts, completeBody, &response)- if err != nil && res == nil {- return false, err- }-- if res.StatusCode == http.StatusTooManyRequests {- return true, err // Retry on 429+ if err != nil {+ return shouldRetry(ctx, res, err)}if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated {body, _ := io.ReadAll(res.Body)- return false, fmt.Errorf("complete multipart failed with status %d: %s", res.StatusCode, string(body))+ return fserrors.ShouldRetryHTTP(res, retryErrorCodes), fmt.Errorf("complete multipart failed with status %d: %s", res.StatusCode, string(body))}return false, nil