Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

azureblob: fix spurious vfs cache corruption errors during chunked reads - fixes #9782

Nick Craig-Woodbee45bccfd54
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 32b834754..642d34d2e 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -19,7 +19,6 @@ import (
"path"
"slices"
"sort"
- "strconv"
"strings"
"sync"
"sync/atomic"
@@ -51,6 +50,7 @@ import (
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/readers"
+ "github.com/rclone/rclone/lib/rest"
"github.com/rclone/rclone/lib/transferaccounter"
"golang.org/x/sync/errgroup"
)
@@ -2345,6 +2345,16 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
} else {
size = *info.ContentLength
}
+ // On a range request Content-Length is the length of the range, not the object, so take the
+ // object's size from the total in Content-Range instead.
+ if info.ContentRange != nil {
+ contentRange, err := rest.ParseContentRange(*info.ContentRange)
+ if err != nil {
+ fs.Debugf(o, "Failed to parse Content-Range %q: %v", *info.ContentRange, err)
+ } else if contentRange.Size >= 0 {
+ size = contentRange.Size
+ }
+ }
if isDirectoryMarker(size, metadata, o.remote) {
return fs.ErrorNotAFile
}
@@ -2378,22 +2388,6 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
// o.accessTier = blob.AccessTier(*info.AccessTier)
// }
o.setMetadata(metadata)
-
- // If it was a Range request, the size is wrong, so correct it
- if info.ContentRange != nil {
- contentRange := *info.ContentRange
- slash := strings.IndexRune(contentRange, '/')
- if slash >= 0 {
- i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
- if err == nil {
- o.size = i
- } else {
- fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
- }
- } else {
- fs.Debugf(o, "Failed to find length in %q", contentRange)
- }
- }
o.contentEncoding = info.ContentEncoding
// If decompressing then size and md5sum are unknown
diff --git a/backend/azureblob/azureblob_internal_test.go b/backend/azureblob/azureblob_internal_test.go
index c234463c2..f8deb0fdb 100644
--- a/backend/azureblob/azureblob_internal_test.go
+++ b/backend/azureblob/azureblob_internal_test.go
@@ -54,6 +54,52 @@ func TestBlockIDCreator(t *testing.T) {
assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
}
+func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
+ pInt64 := func(i int64) *int64 { return &i }
+ newTestObject := func() *Object {
+ return &Object{
+ fs: &Fs{},
+ remote: "test.bin",
+ size: -2, // sentinel to check it gets set
+ }
+ }
+
+ t.Run("WholeBlob", func(t *testing.T) {
+ o := newTestObject()
+ info := blob.DownloadStreamResponse{}
+ info.ContentLength = pInt64(12345)
+ require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+ assert.Equal(t, int64(12345), o.size)
+ })
+
+ t.Run("RangeRequest", func(t *testing.T) {
+ // On a ranged download Content-Length is the chunk length so the
+ // size must come from the total in Content-Range
+ o := newTestObject()
+ info := blob.DownloadStreamResponse{}
+ info.ContentLength = pInt64(67108864)
+ info.ContentRange = pString("bytes 67108864-134217727/169721004032")
+ require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+ assert.Equal(t, int64(169721004032), o.size)
+ })
+
+ t.Run("BadContentRange", func(t *testing.T) {
+ o := newTestObject()
+ info := blob.DownloadStreamResponse{}
+ info.ContentLength = pInt64(67108864)
+ info.ContentRange = pString("potato")
+ require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+ assert.Equal(t, int64(67108864), o.size)
+ })
+
+ t.Run("UnknownLength", func(t *testing.T) {
+ o := newTestObject()
+ info := blob.DownloadStreamResponse{}
+ require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+ assert.Equal(t, int64(-1), o.size)
+ })
+}
+
func TestCopySASTimingConstants(t *testing.T) {
require.Greater(t, sasCopyStartSkew, time.Duration(0))
require.Greater(t, sasCopyValidity, sasCopyStartSkew)