Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: fall back when source ignores ranges

Morax9f93da3299f6
diff --git a/backend/webdav/webdav_internal_test.go b/backend/webdav/webdav_internal_test.go
index dfdd38ef5..42ccbc6d2 100644
--- a/backend/webdav/webdav_internal_test.go
+++ b/backend/webdav/webdav_internal_test.go
@@ -10,11 +10,13 @@ import (
"sync/atomic"
"testing"
+ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/backend/webdav"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/obscure"
+ "github.com/rclone/rclone/fs/operations"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -240,3 +242,45 @@ func TestOpenRetriesMismatchedContentRange(t *testing.T) {
assert.Equal(t, "cde", string(contents))
assert.Equal(t, int32(2), getRequests.Load())
}
+
+func TestCopyFallsBackWhenRangeIgnored(t *testing.T) {
+ var rangeRequests atomic.Int32
+ var fullRequests atomic.Int32
+ ctx, ci := fs.AddConfig(context.Background())
+ ci.LowLevelRetries = 2
+ ci.MultiThreadStreams = 2
+ ci.MultiThreadSet = true
+ ci.MultiThreadCutoff = 1
+ ci.MultiThreadChunkSize = 4
+
+ src, tidy := prepareFileObject(ctx, t, func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Range") == "" {
+ fullRequests.Add(1)
+ } else {
+ rangeRequests.Add(1)
+ }
+ w.Header().Set("Content-Length", "10")
+ w.WriteHeader(http.StatusOK)
+ _, err := io.WriteString(w, "abcdefghij")
+ require.NoError(t, err)
+ })
+ defer tidy()
+
+ dstFs, err := local.NewFs(ctx, "local", t.TempDir(), configmap.Simple{
+ "no_preallocate": "true",
+ "no_sparse": "true",
+ })
+ require.NoError(t, err)
+ dst, err := operations.Copy(ctx, dstFs, nil, "file.txt", src)
+ require.NoError(t, err)
+
+ in, err := dst.Open(ctx)
+ require.NoError(t, err)
+ defer func() { require.NoError(t, in.Close()) }()
+ contents, err := io.ReadAll(in)
+ require.NoError(t, err)
+ assert.Equal(t, "abcdefghij", string(contents))
+ assert.Positive(t, rangeRequests.Load())
+ assert.LessOrEqual(t, rangeRequests.Load(), int32(3))
+ assert.Equal(t, int32(1), fullRequests.Load())
+}
diff --git a/fs/operations/copy.go b/fs/operations/copy.go
index 094aa288c..5e506adef 100644
--- a/fs/operations/copy.go
+++ b/fs/operations/copy.go
@@ -266,7 +266,12 @@ func (c *copy) manualCopy(ctx context.Context) (actionTaken string, newDst fs.Ob
}
if doMultiThreadCopy(ctx, c.f, c.src) {
- return c.multiThreadCopy(ctx, uploadOptions)
+ actionTaken, newDst, err = c.multiThreadCopy(ctx, uploadOptions)
+ if !errors.Is(err, fs.ErrorRangeIgnored) {
+ return actionTaken, newDst, err
+ }
+ fs.Logf(c.src, "multi-thread copy: %v: downloading in a single stream", err)
+ c.tr.Reset(ctx)
}
var in io.ReadCloser