Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

oracleobjectstorage: add --oos-decompress flag to download gzip-encoded files

Nick Craig-Woodb3a41206da61
diff --git a/backend/oracleobjectstorage/client.go b/backend/oracleobjectstorage/client.go
index 0749c4699..e44b35f22 100644
--- a/backend/oracleobjectstorage/client.go
+++ b/backend/oracleobjectstorage/client.go
@@ -90,6 +90,12 @@ func modifyClient(ctx context.Context, opt *Options, client *common.BaseClient)
if opt.Provider == noAuth {
client.Signer = getNoAuthSigner()
}
+ // Set Accept-Encoding: gzip on every request to stop the Go HTTP
+ // transport transparently decompressing objects.
+ client.Interceptor = func(request *http.Request) error {
+ request.Header.Set("Accept-Encoding", "gzip")
+ return nil
+ }
}
// getClient makes http client according to the global options
diff --git a/backend/oracleobjectstorage/client_test.go b/backend/oracleobjectstorage/client_test.go
new file mode 100644
index 000000000..9ef5c2068
--- /dev/null
+++ b/backend/oracleobjectstorage/client_test.go
@@ -0,0 +1,105 @@
+//go:build !plan9 && !solaris && !js
+
+package oracleobjectstorage
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "crypto/md5"
+ "encoding/base64"
+ "encoding/hex"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// TestGzipEncoding checks the handling of objects stored with
+// Content-Encoding: gzip.
+//
+// By default they should be downloaded as stored, not transparently
+// decompressed by the Go runtime. With decompress set they should be
+// decompressed on download with size and hash unknown.
+func TestGzipEncoding(t *testing.T) {
+ ctx := context.Background()
+
+ // Gzip compressed test data served with Content-Encoding: gzip
+ // as if it had been uploaded with that metadata set.
+ plain := []byte("hello, world - some uncompressed data which is longer than the compressed version")
+ var buf bytes.Buffer
+ zw := gzip.NewWriter(&buf)
+ _, err := zw.Write(plain)
+ require.NoError(t, err)
+ require.NoError(t, zw.Close())
+ compressed := buf.Bytes()
+ compressedMd5 := md5.Sum(compressed)
+
+ var gotAcceptEncoding string
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == "GET" {
+ gotAcceptEncoding = r.Header.Get("Accept-Encoding")
+ }
+ w.Header().Set("Content-Encoding", "gzip")
+ w.Header().Set("Content-Length", strconv.Itoa(len(compressed)))
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-MD5", base64.StdEncoding.EncodeToString(compressedMd5[:]))
+ w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
+ _, _ = w.Write(compressed)
+ }))
+ defer srv.Close()
+
+ newFs := func(t *testing.T, extraConfig configmap.Simple) *Fs {
+ m := configmap.Simple{
+ "provider": noAuth,
+ "namespace": "test",
+ "endpoint": srv.URL,
+ }
+ for k, v := range extraConfig {
+ m[k] = v
+ }
+ fsInfo, err := NewFs(ctx, "TestOOSGzip", "bucket", m)
+ require.NoError(t, err)
+ return fsInfo.(*Fs)
+ }
+
+ readObject := func(t *testing.T, f *Fs) ([]byte, *Object) {
+ obj, err := f.NewObject(ctx, "test.gz")
+ require.NoError(t, err)
+ rc, err := obj.Open(ctx)
+ require.NoError(t, err)
+ body, err := io.ReadAll(rc)
+ require.NoError(t, err)
+ require.NoError(t, rc.Close())
+ return body, obj.(*Object)
+ }
+
+ t.Run("Default", func(t *testing.T) {
+ f := newFs(t, configmap.Simple{})
+ body, obj := readObject(t, f)
+ assert.Equal(t, "gzip", gotAcceptEncoding)
+ assert.Equal(t, compressed, body)
+ assert.Equal(t, int64(len(compressed)), obj.Size())
+ md5sum, err := obj.Hash(ctx, hash.MD5)
+ require.NoError(t, err)
+ assert.Equal(t, hex.EncodeToString(compressedMd5[:]), md5sum)
+ })
+
+ t.Run("Decompress", func(t *testing.T) {
+ f := newFs(t, configmap.Simple{"decompress": "true"})
+ body, obj := readObject(t, f)
+ assert.Equal(t, "gzip", gotAcceptEncoding)
+ assert.Equal(t, plain, body)
+ assert.Equal(t, int64(-1), obj.Size())
+ md5sum, err := obj.Hash(ctx, hash.MD5)
+ require.NoError(t, err)
+ assert.Equal(t, "", md5sum)
+ })
+}
diff --git a/backend/oracleobjectstorage/object.go b/backend/oracleobjectstorage/object.go
index 74591c56c..b9c6d1b27 100644
--- a/backend/oracleobjectstorage/object.go
+++ b/backend/oracleobjectstorage/object.go
@@ -21,6 +21,7 @@ import (
"github.com/oracle/oci-go-sdk/v65/objectstorage"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/lib/readers"
)
// ------------------------------------------------------------
@@ -110,6 +111,7 @@ func (o *Object) decodeMetaDataHead(info *objectstorage.HeadObjectResponse) (err
return o.setMetaData(
info.ContentLength,
info.ContentMd5,
+ info.ContentEncoding,
info.ContentType,
info.LastModified,
info.StorageTier,
@@ -120,6 +122,7 @@ func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (er
return o.setMetaData(
info.ContentLength,
info.ContentMd5,
+ info.ContentEncoding,
info.ContentType,
info.LastModified,
info.StorageTier,
@@ -129,6 +132,7 @@ func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (er
func (o *Object) setMetaData(
contentLength *int64,
contentMd5 *string,
+ contentEncoding *string,
contentType *string,
lastModified *common.SDKTime,
storageTier any,
@@ -169,6 +173,11 @@ func (o *Object) setMetaData(
tier := strings.ToLower(fmt.Sprintf("%v", storageTier))
o.storageTier = storageTierMap[tier]
}
+ // If decompressing then size and md5sum are unknown
+ if o.fs.opt.Decompress && contentEncoding != nil && *contentEncoding == "gzip" {
+ o.bytes = -1
+ o.md5 = ""
+ }
return nil
}
@@ -362,6 +371,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
if err != nil {
return nil, err
}
+ // Decompress body if necessary
+ if resp.ContentEncoding != nil && *resp.ContentEncoding == "gzip" {
+ if o.fs.opt.Decompress {
+ return readers.NewGzipReader(resp.HTTPResponse().Body)
+ }
+ o.fs.warnCompressed.Do(func() {
+ fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --oos-decompress to override")
+ })
+ }
if bytes != nil {
o.bytes = *bytes
} else {
diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go
index f24a4d156..a1301ab10 100644
--- a/backend/oracleobjectstorage/options.go
+++ b/backend/oracleobjectstorage/options.go
@@ -72,6 +72,7 @@ type Options struct {
SSECustomerKey string `config:"sse_customer_key"`
SSECustomerKeyFile string `config:"sse_customer_key_file"`
SSECustomerKeySha256 string `config:"sse_customer_key_sha256"`
+ Decompress bool `config:"decompress"`
}
func newOptions() []fs.Option {
@@ -346,5 +347,20 @@ Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Co
Value: sseDefaultAlgorithm,
Help: sseDefaultAlgorithm,
}},
+ }, {
+ Name: "decompress",
+ Help: `If set this will decompress gzip encoded objects.
+
+It is possible to upload objects to object storage with
+"Content-Encoding: gzip" set. Normally rclone will download these
+files as compressed objects.
+
+If this flag is set then rclone will decompress these files with
+"Content-Encoding: gzip" as they are received. This means that rclone
+can't check the size and hash but the file contents will be
+decompressed.
+`,
+ Default: false,
+ Advanced: true,
}}
}
diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go
index fca03bbe9..933662697 100644
--- a/backend/oracleobjectstorage/oracleobjectstorage.go
+++ b/backend/oracleobjectstorage/oracleobjectstorage.go
@@ -10,6 +10,7 @@ import (
"net/http"
"path"
"strings"
+ "sync"
"time"
"github.com/ncw/swift/v2"
@@ -76,16 +77,17 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
// Fs represents a remote object storage server
type Fs struct {
- name string // name of this remote
- root string // the path we are working on if any
- opt Options // parsed config options
- ci *fs.ConfigInfo // global config
- features *fs.Features // optional features
- srv *objectstorage.ObjectStorageClient // the connection to the object storage
- rootBucket string // bucket part of root (if any)
- rootDirectory string // directory part of root (if any)
- cache *bucket.Cache // cache for bucket creation status
- pacer *fs.Pacer // To pace the API calls
+ name string // name of this remote
+ root string // the path we are working on if any
+ opt Options // parsed config options
+ ci *fs.ConfigInfo // global config
+ features *fs.Features // optional features
+ srv *objectstorage.ObjectStorageClient // the connection to the object storage
+ rootBucket string // bucket part of root (if any)
+ rootDirectory string // directory part of root (if any)
+ cache *bucket.Cache // cache for bucket creation status
+ pacer *fs.Pacer // To pace the API calls
+ warnCompressed sync.Once // warn once about compressed files
}
// NewFs Initialize backend
diff --git a/backend/oracleobjectstorage/oracleobjectstorage_test.go b/backend/oracleobjectstorage/oracleobjectstorage_test.go
index f41dfb605..feb215db5 100644
--- a/backend/oracleobjectstorage/oracleobjectstorage_test.go
+++ b/backend/oracleobjectstorage/oracleobjectstorage_test.go
@@ -3,10 +3,20 @@
package oracleobjectstorage
import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "crypto/md5"
+ "fmt"
"testing"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
+ "github.com/rclone/rclone/lib/random"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
// TestIntegration runs integration tests against the remote
@@ -21,6 +31,61 @@ func TestIntegration(t *testing.T) {
})
}
+func gz(t *testing.T, s string) string {
+ var buf bytes.Buffer
+ zw := gzip.NewWriter(&buf)
+ _, err := zw.Write([]byte(s))
+ require.NoError(t, err)
+ err = zw.Close()
+ require.NoError(t, err)
+ return buf.String()
+}
+
+func md5sum(t *testing.T, s string) string {
+ hash := md5.Sum([]byte(s))
+ return fmt.Sprintf("%x", hash)
+}
+
+// InternalTestGzipEncoding tests that a file uploaded with
+// Content-Encoding: gzip can be downloaded with and without
+// decompression.
+func (f *Fs) InternalTestGzipEncoding(t *testing.T) {
+ ctx := context.Background()
+ original := random.String(1000)
+ contents := gz(t, original)
+ item := fstest.NewItem("test-gzip-encoding", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+ obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", nil, &fs.HTTPOption{Key: "Content-Encoding", Value: "gzip"})
+ defer func() {
+ assert.NoError(t, obj.Remove(ctx))
+ }()
+ o := obj.(*Object)
+
+ checkDownload := func(wantContents string, wantSize int64, wantHash string) {
+ gotContents := fstests.ReadObject(ctx, t, o, -1)
+ assert.Equal(t, wantContents, gotContents)
+ assert.Equal(t, wantSize, o.Size())
+ gotHash, err := o.Hash(ctx, hash.MD5)
+ require.NoError(t, err)
+ assert.Equal(t, wantHash, gotHash)
+ }
+
+ t.Run("NoDecompress", func(t *testing.T) {
+ checkDownload(contents, int64(len(contents)), md5sum(t, contents))
+ })
+ t.Run("Decompress", func(t *testing.T) {
+ f.opt.Decompress = true
+ defer func() {
+ f.opt.Decompress = false
+ }()
+ checkDownload(original, -1, "")
+ })
+}
+
+// InternalTest is called by fstests.Run to extra tests
+func (f *Fs) InternalTest(t *testing.T) {
+ t.Run("GzipEncoding", f.InternalTestGzipEncoding)
+}
+
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}