"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
webdav: retry Nextcloud chunk uploads individually - fixes #10005
Stochastic039b69adb8a3
diff --git a/backend/webdav/chunking.go b/backend/webdav/chunking.goindex ce29507c9..a871353c3 100644--- a/backend/webdav/chunking.go+++ b/backend/webdav/chunking.go@@ -17,6 +17,7 @@ import ("time""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/lib/readers""github.com/rclone/rclone/lib/rest")@@ -133,7 +134,26 @@ func (o *Object) uploadChunks(ctx context.Context, in0 io.Reader, size int64, pareturn io.NopCloser(in), nil}- err := partObj.updateSimple(ctx, in, getBody, partObj.remote, contentLength, "application/x-www-form-urlencoded", nil, o.fs.chunksUploadURL, options...)+ // Retry this chunk in place so a transient failure does not make the caller+ // restart the whole upload and resend chunks that were already accepted.+ maxTries := max(fs.GetConfig(ctx).LowLevelRetries, 1)+ var err error+ for try := 1; try <= maxTries; try++ {+ retryBody, bodyErr := getBody()+ if bodyErr != nil {+ return fmt.Errorf("rewinding chunk for retry failed: %w", bodyErr)+ }+ err = partObj.updateSimple(ctx, retryBody, getBody, partObj.remote, contentLength, "application/x-www-form-urlencoded", nil, o.fs.chunksUploadURL, options...)+ _ = retryBody.Close()+ if err == nil || (!fserrors.IsRetryError(err) && !fserrors.ShouldRetry(err)) {+ break+ }+ if try == maxTries {+ // The chunk retry budget is exhausted. Do not let the outer+ // low-level retry restart the upload from its first chunk.+ err = fserrors.NoLowLevelRetryError(errors.New(err.Error()))+ }+ }if err != nil {return fmt.Errorf("uploading chunk failed: %w", err)}diff --git a/backend/webdav/webdav_internal_test.go b/backend/webdav/webdav_internal_test.goindex f5de11d8b..87f223120 100644--- a/backend/webdav/webdav_internal_test.go+++ b/backend/webdav/webdav_internal_test.go@@ -1,6 +1,7 @@package webdav_testimport (+ "bytes""context""fmt""io"@@ -9,6 +10,7 @@ import ("strings""sync/atomic""testing"+ "time""github.com/rclone/rclone/backend/local""github.com/rclone/rclone/backend/webdav"@@ -16,11 +18,93 @@ import ("github.com/rclone/rclone/fs/config/configfile""github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/config/obscure"+ "github.com/rclone/rclone/fs/object""github.com/rclone/rclone/fs/operations""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")+func TestChunkedUploadRetriesOnlyFailedChunk(t *testing.T) {+ tests := []struct {+ name string+ failChunk1 bool+ wantError bool+ wantChunk2 int32+ }{+ {name: "retry failed chunk", wantChunk2: 1},+ {name: "stop after chunk retry budget", failChunk1: true, wantError: true},+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ var chunk0, chunk1, chunk2 atomic.Int32+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/dav/uploads/alice/"):+ switch {+ case strings.HasSuffix(r.URL.Path, "000000000000000-000000000000003"):+ chunk0.Add(1)+ case strings.HasSuffix(r.URL.Path, "000000000000004-000000000000007"):+ if test.failChunk1 || chunk1.Add(1) == 1 {+ if test.failChunk1 {+ chunk1.Add(1)+ }+ w.WriteHeader(http.StatusServiceUnavailable)+ return+ }+ case strings.HasSuffix(r.URL.Path, "000000000000008-000000000000009"):+ chunk2.Add(1)+ default:+ t.Errorf("unexpected chunk path %q", r.URL.Path)+ }+ w.WriteHeader(http.StatusCreated)+ case r.Method == "MKCOL":+ w.WriteHeader(http.StatusCreated)+ case r.Method == "MOVE":+ w.WriteHeader(http.StatusCreated)+ case r.Method == "DELETE":+ w.WriteHeader(http.StatusNotFound)+ case r.Method == "PROPFIND":+ w.WriteHeader(http.StatusMultiStatus)+ _, _ = io.WriteString(w, `<d:multistatus xmlns:d="DAV:"><d:response><d:href>/file</d:href><d:propstat><d:prop><d:getcontentlength>10</d:getcontentlength><d:resourcetype/></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>`)+ case r.Method == "PATCH":+ w.Header().Set("OC-Checksum", "SHA1:0000000000000000000000000000000000000000")+ w.WriteHeader(http.StatusOK)+ default:+ t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)+ w.WriteHeader(http.StatusNotFound)+ }+ })+ ts := httptest.NewServer(handler)+ defer ts.Close()+ configfile.Install()+ ctx, ci := fs.AddConfig(context.Background())+ ci.LowLevelRetries = 2+ m := configmap.Simple{+ "type": "webdav",+ "url": ts.URL + "/remote.php/dav/files/alice",+ "vendor": "nextcloud",+ "nextcloud_chunk_size": "4B",+ }+ f, err := webdav.NewFs(ctx, remoteName, "", m)+ require.NoError(t, err)+ src := object.NewStaticObjectInfo("file", time.Now(), 10, true, nil, f)+ body := []byte("abcdefghij")+ err = operations.Retry(ctx, src, ci.LowLevelRetries, func() error {+ _, err := f.Put(ctx, bytes.NewReader(body), src)+ return err+ })+ if test.wantError {+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ }+ assert.EqualValues(t, 1, chunk0.Load(), "successful first chunk should not be uploaded again")+ assert.EqualValues(t, 2, chunk1.Load(), "only the failed chunk should be retried")+ assert.EqualValues(t, test.wantChunk2, chunk2.Load(), "later chunks should not upload after failure")+ })+ }+}+var (remoteName = "TestWebDAV"headers = []string{"X-Potato", "sausage", "X-Rhubarb", "cucumber"}