"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: fall back when source ignores ranges
Morax9f93da3299f6
diff --git a/backend/webdav/webdav_internal_test.go b/backend/webdav/webdav_internal_test.goindex dfdd38ef5..42ccbc6d2 100644--- a/backend/webdav/webdav_internal_test.go+++ b/backend/webdav/webdav_internal_test.go@@ -10,11 +10,13 @@ import ("sync/atomic""testing"+ "github.com/rclone/rclone/backend/local""github.com/rclone/rclone/backend/webdav""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/config/configfile""github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/config/obscure"+ "github.com/rclone/rclone/fs/operations""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -240,3 +242,45 @@ func TestOpenRetriesMismatchedContentRange(t *testing.T) {assert.Equal(t, "cde", string(contents))assert.Equal(t, int32(2), getRequests.Load())}++func TestCopyFallsBackWhenRangeIgnored(t *testing.T) {+ var rangeRequests atomic.Int32+ var fullRequests atomic.Int32+ ctx, ci := fs.AddConfig(context.Background())+ ci.LowLevelRetries = 2+ ci.MultiThreadStreams = 2+ ci.MultiThreadSet = true+ ci.MultiThreadCutoff = 1+ ci.MultiThreadChunkSize = 4++ src, tidy := prepareFileObject(ctx, t, func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Range") == "" {+ fullRequests.Add(1)+ } else {+ rangeRequests.Add(1)+ }+ w.Header().Set("Content-Length", "10")+ w.WriteHeader(http.StatusOK)+ _, err := io.WriteString(w, "abcdefghij")+ require.NoError(t, err)+ })+ defer tidy()++ dstFs, err := local.NewFs(ctx, "local", t.TempDir(), configmap.Simple{+ "no_preallocate": "true",+ "no_sparse": "true",+ })+ require.NoError(t, err)+ dst, err := operations.Copy(ctx, dstFs, nil, "file.txt", src)+ require.NoError(t, err)++ in, err := dst.Open(ctx)+ require.NoError(t, err)+ defer func() { require.NoError(t, in.Close()) }()+ contents, err := io.ReadAll(in)+ require.NoError(t, err)+ assert.Equal(t, "abcdefghij", string(contents))+ assert.Positive(t, rangeRequests.Load())+ assert.LessOrEqual(t, rangeRequests.Load(), int32(3))+ assert.Equal(t, int32(1), fullRequests.Load())+}diff --git a/fs/operations/copy.go b/fs/operations/copy.goindex 094aa288c..5e506adef 100644--- a/fs/operations/copy.go+++ b/fs/operations/copy.go@@ -266,7 +266,12 @@ func (c *copy) manualCopy(ctx context.Context) (actionTaken string, newDst fs.Ob}if doMultiThreadCopy(ctx, c.f, c.src) {- return c.multiThreadCopy(ctx, uploadOptions)+ actionTaken, newDst, err = c.multiThreadCopy(ctx, uploadOptions)+ if !errors.Is(err, fs.ErrorRangeIgnored) {+ return actionTaken, newDst, err+ }+ fs.Logf(c.src, "multi-thread copy: %v: downloading in a single stream", err)+ c.tr.Reset(ctx)}var in io.ReadCloser