Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

shade: retry server errors instead of failing the transfer

Nick Craig-Wooda4972de505af
diff --git a/backend/shade/shade.go b/backend/shade/shade.go
index dc8d1c64e..eacbdc37b 100644
--- a/backend/shade/shade.go
+++ b/backend/shade/shade.go
@@ -22,6 +22,7 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
+ "github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
@@ -42,6 +43,25 @@ const (
maxUploadParts = 10000 // maximum allowed number of parts in a multipart upload
)
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+ 429, // Too Many Requests.
+ 500, // Internal Server Error
+ 502, // Bad Gateway
+ 503, // Service Unavailable
+ 504, // Gateway Timeout
+ 509, // Bandwidth Limit Exceeded
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried. It returns the err as a convenience
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+ if fserrors.ContextError(ctx, &err) {
+ return false, err
+ }
+ return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
@@ -126,14 +146,14 @@ func (f *Fs) refreshJWTToken(ctx context.Context) (string, error) {
res, err := f.apiSrv.Call(ctx, &opts)
if err != nil {
fs.Debugf(f, "Token request failed: %v", err)
- return false, err
+ return shouldRetry(ctx, res, err)
}
defer fs.CheckClose(res.Body, &err)
if res.StatusCode != http.StatusOK {
fs.Debugf(f, "Token request failed with code: %d", res.StatusCode)
- return res.StatusCode == http.StatusTooManyRequests, fmt.Errorf("failed to get ShadeFS token, status: %d", res.StatusCode)
+ return fserrors.ShouldRetryHTTP(res, retryErrorCodes), fmt.Errorf("failed to get ShadeFS token, status: %d", res.StatusCode)
}
// Read token directly as plain text
@@ -202,10 +222,7 @@ func (f *Fs) callAPI(ctx context.Context, method, path string, response any) (*h
} else {
res, err = f.srv.Call(ctx, &opts)
}
- if err != nil {
- return res != nil && res.StatusCode == http.StatusTooManyRequests, err
- }
- return false, nil
+ return shouldRetry(ctx, res, err)
})
return res, err
}
@@ -334,11 +351,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
err = o.fs.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(ctx, &opts)
- if err != nil && resp.StatusCode == http.StatusBadRequest {
+ if err != nil && resp != nil && resp.StatusCode == http.StatusBadRequest {
fs.Debugf(f, "Bad token from server: %v", token)
}
- return resp != nil && resp.StatusCode == http.StatusTooManyRequests, err
+ return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
@@ -860,9 +877,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
}
res, err = client.Do(req)
if err != nil {
- return false, err
+ return shouldRetry(ctx, nil, err)
}
- return res.StatusCode == http.StatusTooManyRequests, nil
+ return fserrors.ShouldRetryHTTP(res, retryErrorCodes), nil
})
if err != nil {
@@ -899,12 +916,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
err = o.fs.pacer.Call(func() (bool, error) {
downloadRes, err = client.Call(ctx, &opts)
if err != nil {
- return false, err
+ return shouldRetry(ctx, downloadRes, err)
}
if downloadRes == nil {
return false, fmt.Errorf("failed to fetch presigned URL")
}
- return downloadRes.StatusCode == http.StatusTooManyRequests, nil
+ return fserrors.ShouldRetryHTTP(downloadRes, retryErrorCodes), nil
})
if err != nil {
diff --git a/backend/shade/upload.go b/backend/shade/upload.go
index 0419f42d1..89f62bac2 100644
--- a/backend/shade/upload.go
+++ b/backend/shade/upload.go
@@ -17,6 +17,7 @@ import (
"github.com/rclone/rclone/backend/shade/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
+ "github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/rest"
)
@@ -125,10 +126,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
err = o.fs.pacer.Call(func() (bool, error) {
res, err := o.fs.srv.CallJSON(ctx, &opts, reqBody, &initResp)
- if err != nil {
- return res != nil && res.StatusCode == http.StatusTooManyRequests, err
- }
- return false, nil
+ return shouldRetry(ctx, res, err)
})
if err != nil {
@@ -183,10 +181,7 @@ func (s *shadeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, read
err = s.f.pacer.Call(func() (bool, error) {
res, err := s.f.srv.CallJSON(ctx, &partOpts, nil, &partURL)
- if err != nil {
- return res != nil && res.StatusCode == http.StatusTooManyRequests, err
- }
- return false, nil
+ return shouldRetry(ctx, res, err)
})
if err != nil {
@@ -195,7 +190,6 @@ func (s *shadeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, read
opts := rest.Opts{
Method: "PUT",
RootURL: partURL.URL,
- Body: &chunk,
ContentType: "",
ContentLength: &n,
}
@@ -208,11 +202,10 @@ func (s *shadeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, read
}
err = s.f.pacer.Call(func() (bool, error) {
+ // Use a fresh reader for each attempt so retries resend the whole chunk
+ opts.Body = bytes.NewReader(chunk.Bytes())
uploadRes, err = s.f.srv.Call(ctx, &opts)
- if err != nil {
- return uploadRes != nil && uploadRes.StatusCode == http.StatusTooManyRequests, err
- }
- return false, nil
+ return shouldRetry(ctx, uploadRes, err)
})
if err != nil {
@@ -276,17 +269,13 @@ func (s *shadeChunkWriter) Close(ctx context.Context) error {
err = s.f.pacer.Call(func() (bool, error) {
res, err := s.f.srv.CallJSON(ctx, &completeOpts, completeBody, &response)
- if err != nil && res == nil {
- return false, err
- }
-
- if res.StatusCode == http.StatusTooManyRequests {
- return true, err // Retry on 429
+ if err != nil {
+ return shouldRetry(ctx, res, err)
}
if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated {
body, _ := io.ReadAll(res.Body)
- return false, fmt.Errorf("complete multipart failed with status %d: %s", res.StatusCode, string(body))
+ return fserrors.ShouldRetryHTTP(res, retryErrorCodes), fmt.Errorf("complete multipart failed with status %d: %s", res.StatusCode, string(body))
}
return false, nil