Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

compress: fix crash on ranged reads when gzip metadata is corrupted

Acts1631c8d60a67fc53
diff --git a/backend/compress/compress.go b/backend/compress/compress.go
index 3de158897..770421974 100644
--- a/backend/compress/compress.go
+++ b/backend/compress/compress.go
@@ -1085,6 +1085,35 @@ type ObjectMetadata struct {
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
}
+// validate checks metadata read from the wrapped remote before it is used to
+// construct a compressed reader.
+func (meta *ObjectMetadata) validate() error {
+ if meta.Mode != Gzip {
+ return nil
+ }
+ if meta.CompressionMetadataGzip == nil {
+ return errors.New("missing gzip metadata")
+ }
+ if meta.CompressionMetadataGzip.BlockSize <= 0 {
+ return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
+ }
+ if meta.Size != meta.CompressionMetadataGzip.Size {
+ return errors.New("gzip metadata size does not match object size")
+ }
+ if meta.Size < 0 {
+ return errors.New("invalid gzip object size")
+ }
+ blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
+ blocks := meta.Size / blockSize
+ if meta.Size%blockSize != 0 {
+ blocks++
+ }
+ if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
+ return errors.New("gzip block data is incomplete")
+ }
+ return nil
+}
+
// Object with external metadata
type Object struct {
fs.Object // Wraps around data object for this object
@@ -1108,6 +1137,9 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
if err = jr.Decode(meta); err != nil {
return nil, err
}
+ if err = meta.validate(); err != nil {
+ return nil, err
+ }
return meta, nil
}
diff --git a/backend/compress/compress_test.go b/backend/compress/compress_test.go
index 000d70330..c78b3afb7 100644
--- a/backend/compress/compress_test.go
+++ b/backend/compress/compress_test.go
@@ -6,6 +6,8 @@ import (
"path/filepath"
"testing"
+ "github.com/buengese/sgzip"
+
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/local"
_ "github.com/rclone/rclone/backend/s3"
@@ -14,6 +16,93 @@ import (
"github.com/rclone/rclone/fstest/fstests"
)
+func TestObjectMetadataValidate(t *testing.T) {
+ tests := []struct {
+ name string
+ meta ObjectMetadata
+ want bool
+ }{
+ {
+ name: "nil gzip metadata",
+ meta: ObjectMetadata{Mode: Gzip},
+ want: true,
+ },
+ {
+ name: "zero block size",
+ meta: ObjectMetadata{
+ Mode: Gzip,
+ Size: 1024,
+ CompressionMetadataGzip: &sgzip.GzipMetadata{
+ BlockSize: 0,
+ Size: 1024,
+ BlockData: []uint32{0},
+ },
+ },
+ want: true,
+ },
+ {
+ name: "negative block size",
+ meta: ObjectMetadata{
+ Mode: Gzip,
+ Size: 1024,
+ CompressionMetadataGzip: &sgzip.GzipMetadata{
+ BlockSize: -1,
+ Size: 1024,
+ BlockData: []uint32{0},
+ },
+ },
+ want: true,
+ },
+ {
+ name: "mismatched sizes",
+ meta: ObjectMetadata{
+ Mode: Gzip,
+ Size: 1024,
+ CompressionMetadataGzip: &sgzip.GzipMetadata{
+ BlockSize: 512,
+ Size: 2048,
+ BlockData: []uint32{0, 1},
+ },
+ },
+ want: true,
+ },
+ {
+ name: "short block data",
+ meta: ObjectMetadata{
+ Mode: Gzip,
+ Size: 1025,
+ CompressionMetadataGzip: &sgzip.GzipMetadata{
+ BlockSize: 512,
+ Size: 1025,
+ BlockData: []uint32{0, 1},
+ },
+ },
+ want: true,
+ },
+ {
+ name: "valid",
+ meta: ObjectMetadata{
+ Mode: Gzip,
+ Size: 1025,
+ CompressionMetadataGzip: &sgzip.GzipMetadata{
+ BlockSize: 512,
+ Size: 1025,
+ BlockData: []uint32{0, 1, 2},
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ err := test.meta.validate()
+ if (err != nil) != test.want {
+ t.Fatalf("validate() error = %v, want error: %v", err, test.want)
+ }
+ })
+ }
+}
+
var defaultOpt = fstests.Opt{
RemoteName: "TestCompress:",
NilObject: (*Object)(nil),