"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
oracleobjectstorage: add --oos-decompress flag to download gzip-encoded files
Nick Craig-Woodb3a41206da61
diff --git a/backend/oracleobjectstorage/client.go b/backend/oracleobjectstorage/client.goindex 0749c4699..e44b35f22 100644--- a/backend/oracleobjectstorage/client.go+++ b/backend/oracleobjectstorage/client.go@@ -90,6 +90,12 @@ func modifyClient(ctx context.Context, opt *Options, client *common.BaseClient)if opt.Provider == noAuth {client.Signer = getNoAuthSigner()}+ // Set Accept-Encoding: gzip on every request to stop the Go HTTP+ // transport transparently decompressing objects.+ client.Interceptor = func(request *http.Request) error {+ request.Header.Set("Accept-Encoding", "gzip")+ return nil+ }}// getClient makes http client according to the global optionsdiff --git a/backend/oracleobjectstorage/client_test.go b/backend/oracleobjectstorage/client_test.gonew file mode 100644index 000000000..9ef5c2068--- /dev/null+++ b/backend/oracleobjectstorage/client_test.go@@ -0,0 +1,105 @@+//go:build !plan9 && !solaris && !js++package oracleobjectstorage++import (+ "bytes"+ "compress/gzip"+ "context"+ "crypto/md5"+ "encoding/base64"+ "encoding/hex"+ "io"+ "net/http"+ "net/http/httptest"+ "strconv"+ "testing"+ "time"++ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/hash"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// TestGzipEncoding checks the handling of objects stored with+// Content-Encoding: gzip.+//+// By default they should be downloaded as stored, not transparently+// decompressed by the Go runtime. With decompress set they should be+// decompressed on download with size and hash unknown.+func TestGzipEncoding(t *testing.T) {+ ctx := context.Background()++ // Gzip compressed test data served with Content-Encoding: gzip+ // as if it had been uploaded with that metadata set.+ plain := []byte("hello, world - some uncompressed data which is longer than the compressed version")+ var buf bytes.Buffer+ zw := gzip.NewWriter(&buf)+ _, err := zw.Write(plain)+ require.NoError(t, err)+ require.NoError(t, zw.Close())+ compressed := buf.Bytes()+ compressedMd5 := md5.Sum(compressed)++ var gotAcceptEncoding string+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method == "GET" {+ gotAcceptEncoding = r.Header.Get("Accept-Encoding")+ }+ w.Header().Set("Content-Encoding", "gzip")+ w.Header().Set("Content-Length", strconv.Itoa(len(compressed)))+ w.Header().Set("Content-Type", "application/octet-stream")+ w.Header().Set("Content-MD5", base64.StdEncoding.EncodeToString(compressedMd5[:]))+ w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))+ _, _ = w.Write(compressed)+ }))+ defer srv.Close()++ newFs := func(t *testing.T, extraConfig configmap.Simple) *Fs {+ m := configmap.Simple{+ "provider": noAuth,+ "namespace": "test",+ "endpoint": srv.URL,+ }+ for k, v := range extraConfig {+ m[k] = v+ }+ fsInfo, err := NewFs(ctx, "TestOOSGzip", "bucket", m)+ require.NoError(t, err)+ return fsInfo.(*Fs)+ }++ readObject := func(t *testing.T, f *Fs) ([]byte, *Object) {+ obj, err := f.NewObject(ctx, "test.gz")+ require.NoError(t, err)+ rc, err := obj.Open(ctx)+ require.NoError(t, err)+ body, err := io.ReadAll(rc)+ require.NoError(t, err)+ require.NoError(t, rc.Close())+ return body, obj.(*Object)+ }++ t.Run("Default", func(t *testing.T) {+ f := newFs(t, configmap.Simple{})+ body, obj := readObject(t, f)+ assert.Equal(t, "gzip", gotAcceptEncoding)+ assert.Equal(t, compressed, body)+ assert.Equal(t, int64(len(compressed)), obj.Size())+ md5sum, err := obj.Hash(ctx, hash.MD5)+ require.NoError(t, err)+ assert.Equal(t, hex.EncodeToString(compressedMd5[:]), md5sum)+ })++ t.Run("Decompress", func(t *testing.T) {+ f := newFs(t, configmap.Simple{"decompress": "true"})+ body, obj := readObject(t, f)+ assert.Equal(t, "gzip", gotAcceptEncoding)+ assert.Equal(t, plain, body)+ assert.Equal(t, int64(-1), obj.Size())+ md5sum, err := obj.Hash(ctx, hash.MD5)+ require.NoError(t, err)+ assert.Equal(t, "", md5sum)+ })+}diff --git a/backend/oracleobjectstorage/object.go b/backend/oracleobjectstorage/object.goindex 74591c56c..b9c6d1b27 100644--- a/backend/oracleobjectstorage/object.go+++ b/backend/oracleobjectstorage/object.go@@ -21,6 +21,7 @@ import ("github.com/oracle/oci-go-sdk/v65/objectstorage""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/lib/readers")// ------------------------------------------------------------@@ -110,6 +111,7 @@ func (o *Object) decodeMetaDataHead(info *objectstorage.HeadObjectResponse) (errreturn o.setMetaData(info.ContentLength,info.ContentMd5,+ info.ContentEncoding,info.ContentType,info.LastModified,info.StorageTier,@@ -120,6 +122,7 @@ func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (erreturn o.setMetaData(info.ContentLength,info.ContentMd5,+ info.ContentEncoding,info.ContentType,info.LastModified,info.StorageTier,@@ -129,6 +132,7 @@ func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (erfunc (o *Object) setMetaData(contentLength *int64,contentMd5 *string,+ contentEncoding *string,contentType *string,lastModified *common.SDKTime,storageTier any,@@ -169,6 +173,11 @@ func (o *Object) setMetaData(tier := strings.ToLower(fmt.Sprintf("%v", storageTier))o.storageTier = storageTierMap[tier]}+ // If decompressing then size and md5sum are unknown+ if o.fs.opt.Decompress && contentEncoding != nil && *contentEncoding == "gzip" {+ o.bytes = -1+ o.md5 = ""+ }return nil}@@ -362,6 +371,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloif err != nil {return nil, err}+ // Decompress body if necessary+ if resp.ContentEncoding != nil && *resp.ContentEncoding == "gzip" {+ if o.fs.opt.Decompress {+ return readers.NewGzipReader(resp.HTTPResponse().Body)+ }+ o.fs.warnCompressed.Do(func() {+ fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --oos-decompress to override")+ })+ }if bytes != nil {o.bytes = *bytes} else {diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.goindex f24a4d156..a1301ab10 100644--- a/backend/oracleobjectstorage/options.go+++ b/backend/oracleobjectstorage/options.go@@ -72,6 +72,7 @@ type Options struct {SSECustomerKey string `config:"sse_customer_key"`SSECustomerKeyFile string `config:"sse_customer_key_file"`SSECustomerKeySha256 string `config:"sse_customer_key_sha256"`+ Decompress bool `config:"decompress"`}func newOptions() []fs.Option {@@ -346,5 +347,20 @@ Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/CoValue: sseDefaultAlgorithm,Help: sseDefaultAlgorithm,}},+ }, {+ Name: "decompress",+ Help: `If set this will decompress gzip encoded objects.++It is possible to upload objects to object storage with+"Content-Encoding: gzip" set. Normally rclone will download these+files as compressed objects.++If this flag is set then rclone will decompress these files with+"Content-Encoding: gzip" as they are received. This means that rclone+can't check the size and hash but the file contents will be+decompressed.+`,+ Default: false,+ Advanced: true,}}}diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.goindex fca03bbe9..933662697 100644--- a/backend/oracleobjectstorage/oracleobjectstorage.go+++ b/backend/oracleobjectstorage/oracleobjectstorage.go@@ -10,6 +10,7 @@ import ("net/http""path""strings"+ "sync""time""github.com/ncw/swift/v2"@@ -76,16 +77,17 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{// Fs represents a remote object storage servertype Fs struct {- name string // name of this remote- root string // the path we are working on if any- opt Options // parsed config options- ci *fs.ConfigInfo // global config- features *fs.Features // optional features- srv *objectstorage.ObjectStorageClient // the connection to the object storage- rootBucket string // bucket part of root (if any)- rootDirectory string // directory part of root (if any)- cache *bucket.Cache // cache for bucket creation status- pacer *fs.Pacer // To pace the API calls+ name string // name of this remote+ root string // the path we are working on if any+ opt Options // parsed config options+ ci *fs.ConfigInfo // global config+ features *fs.Features // optional features+ srv *objectstorage.ObjectStorageClient // the connection to the object storage+ rootBucket string // bucket part of root (if any)+ rootDirectory string // directory part of root (if any)+ cache *bucket.Cache // cache for bucket creation status+ pacer *fs.Pacer // To pace the API calls+ warnCompressed sync.Once // warn once about compressed files}// NewFs Initialize backenddiff --git a/backend/oracleobjectstorage/oracleobjectstorage_test.go b/backend/oracleobjectstorage/oracleobjectstorage_test.goindex f41dfb605..feb215db5 100644--- a/backend/oracleobjectstorage/oracleobjectstorage_test.go+++ b/backend/oracleobjectstorage/oracleobjectstorage_test.go@@ -3,10 +3,20 @@package oracleobjectstorageimport (+ "bytes"+ "compress/gzip"+ "context"+ "crypto/md5"+ "fmt""testing""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fstest""github.com/rclone/rclone/fstest/fstests"+ "github.com/rclone/rclone/lib/random"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require")// TestIntegration runs integration tests against the remote@@ -21,6 +31,61 @@ func TestIntegration(t *testing.T) {})}+func gz(t *testing.T, s string) string {+ var buf bytes.Buffer+ zw := gzip.NewWriter(&buf)+ _, err := zw.Write([]byte(s))+ require.NoError(t, err)+ err = zw.Close()+ require.NoError(t, err)+ return buf.String()+}++func md5sum(t *testing.T, s string) string {+ hash := md5.Sum([]byte(s))+ return fmt.Sprintf("%x", hash)+}++// InternalTestGzipEncoding tests that a file uploaded with+// Content-Encoding: gzip can be downloaded with and without+// decompression.+func (f *Fs) InternalTestGzipEncoding(t *testing.T) {+ ctx := context.Background()+ original := random.String(1000)+ contents := gz(t, original)+ item := fstest.NewItem("test-gzip-encoding", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))+ obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", nil, &fs.HTTPOption{Key: "Content-Encoding", Value: "gzip"})+ defer func() {+ assert.NoError(t, obj.Remove(ctx))+ }()+ o := obj.(*Object)++ checkDownload := func(wantContents string, wantSize int64, wantHash string) {+ gotContents := fstests.ReadObject(ctx, t, o, -1)+ assert.Equal(t, wantContents, gotContents)+ assert.Equal(t, wantSize, o.Size())+ gotHash, err := o.Hash(ctx, hash.MD5)+ require.NoError(t, err)+ assert.Equal(t, wantHash, gotHash)+ }++ t.Run("NoDecompress", func(t *testing.T) {+ checkDownload(contents, int64(len(contents)), md5sum(t, contents))+ })+ t.Run("Decompress", func(t *testing.T) {+ f.opt.Decompress = true+ defer func() {+ f.opt.Decompress = false+ }()+ checkDownload(original, -1, "")+ })+}++// InternalTest is called by fstests.Run to extra tests+func (f *Fs) InternalTest(t *testing.T) {+ t.Run("GzipEncoding", f.InternalTestGzipEncoding)+}+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {return f.setUploadChunkSize(cs)}