"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
//go:build !plan9 && !solaris && !js
package oracleobjectstorage
import (
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"io"
"maps"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/hash"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestGzipEncoding checks the handling of objects stored with
// Content-Encoding: gzip.
//
// By default they should be downloaded as stored, not transparently
// decompressed by the Go runtime. With decompress set they should be
// decompressed on download with size and hash unknown.
func TestGzipEncoding(t *testing.T) {
ctx := context.Background()
// Gzip compressed test data served with Content-Encoding: gzip
// as if it had been uploaded with that metadata set.
plain := []byte("hello, world - some uncompressed data which is longer than the compressed version")
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
_, err := zw.Write(plain)
require.NoError(t, err)
require.NoError(t, zw.Close())
compressed := buf.Bytes()
compressedMd5 := md5.Sum(compressed)
var gotAcceptEncoding string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
gotAcceptEncoding = r.Header.Get("Accept-Encoding")
}
w.Header().Set("Content-Encoding", "gzip")
w.Header().Set("Content-Length", strconv.Itoa(len(compressed)))
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-MD5", base64.StdEncoding.EncodeToString(compressedMd5[:]))
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
_, _ = w.Write(compressed)
}))
defer srv.Close()
newFs := func(t *testing.T, extraConfig configmap.Simple) *Fs {
m := configmap.Simple{
"provider": noAuth,
"namespace": "test",
"endpoint": srv.URL,
}
maps.Copy(m, extraConfig)
fsInfo, err := NewFs(ctx, "TestOOSGzip", "bucket", m)
require.NoError(t, err)
return fsInfo.(*Fs)
}
readObject := func(t *testing.T, f *Fs) ([]byte, *Object) {
obj, err := f.NewObject(ctx, "test.gz")
require.NoError(t, err)
rc, err := obj.Open(ctx)
require.NoError(t, err)
body, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
return body, obj.(*Object)
}
t.Run("Default", func(t *testing.T) {
f := newFs(t, configmap.Simple{})
body, obj := readObject(t, f)
assert.Equal(t, "gzip", gotAcceptEncoding)
assert.Equal(t, compressed, body)
assert.Equal(t, int64(len(compressed)), obj.Size())
md5sum, err := obj.Hash(ctx, hash.MD5)
require.NoError(t, err)
assert.Equal(t, hex.EncodeToString(compressedMd5[:]), md5sum)
})
t.Run("Decompress", func(t *testing.T) {
f := newFs(t, configmap.Simple{"decompress": "true"})
body, obj := readObject(t, f)
assert.Equal(t, "gzip", gotAcceptEncoding)
assert.Equal(t, plain, body)
assert.Equal(t, int64(-1), obj.Size())
md5sum, err := obj.Hash(ctx, hash.MD5)
require.NoError(t, err)
assert.Equal(t, "", md5sum)
})
}
// serviceError is a minimal common.ServiceError for shouldRetry
type serviceError struct {
status int
code string
}
func (e serviceError) Error() string { return e.code }
func (e serviceError) GetHTTPStatusCode() int { return e.status }
func (e serviceError) GetMessage() string { return e.code }
func (e serviceError) GetCode() string { return e.code }
func (e serviceError) GetOpcRequestID() string { return "" }
func TestShouldRetryServiceError(t *testing.T) {
ctx := context.Background()
for _, test := range []struct {
status int
code string
want bool
}{
{http.StatusConflict, "ConcurrentObjectUpdate", true},
{http.StatusConflict, "BucketAlreadyExists", false},
{http.StatusRequestTimeout, "RequestTimeout", true},
{http.StatusNotFound, "ObjectNotFound", false},
} {
err := serviceError{status: test.status, code: test.code}
retry, gotErr := shouldRetry(ctx, nil, err)
assert.Equal(t, test.want, retry, test.code)
assert.Equal(t, error(err), gotErr)
}
}