"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
package s3
import (
"context"
"crypto/md5"
"encoding/hex"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/rclone/gofakes3"
_ "github.com/rclone/rclone/backend/crypt"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// newTestBackend serves a root directory containing a bucket directory, an
// object inside that bucket and a root-level file that is not part of any
// bucket. It returns the backend and the serve root path.
func newTestBackend(t *testing.T) (*s3Backend, string) {
fstest.Initialise()
ctx := context.Background()
root := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(root, "bucket"), 0777))
require.NoError(t, os.WriteFile(filepath.Join(root, "root-secret.txt"), []byte(rootSecret), 0666))
require.NoError(t, os.WriteFile(filepath.Join(root, "bucket", "object.txt"), []byte("normal object"), 0666))
f, err := fs.NewFs(ctx, root)
require.NoError(t, err)
opt := Opt
opt.HTTP.ListenAddr = []string{endpoint}
w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)
require.NoError(t, err)
return newBackend(w), root
}
const rootSecret = "ROOT_LEVEL_SECRET_MARKER"
// TestPathTraversal checks that dot-dot segments in an object key cannot
// escape the bucket namespace to read, overwrite or create files outside the
// selected bucket.
func TestPathTraversal(t *testing.T) {
ctx := context.Background()
t.Run("Get", func(t *testing.T) {
b, _ := newTestBackend(t)
_, err := b.GetObject(ctx, "bucket", "../root-secret.txt", nil)
assert.Error(t, err, "GET with dot-dot key must not read the root-level file")
})
t.Run("Head", func(t *testing.T) {
b, _ := newTestBackend(t)
_, err := b.HeadObject(ctx, "bucket", "../root-secret.txt")
assert.Error(t, err, "HEAD with dot-dot key must not find the root-level file")
})
t.Run("Put", func(t *testing.T) {
b, root := newTestBackend(t)
const payload = "OVERWRITTEN_BY_DOTDOT"
_, err := b.PutObject(ctx, "bucket", "../root-secret.txt", map[string]string{}, strings.NewReader(payload), int64(len(payload)))
assert.Error(t, err, "PUT with dot-dot key must not overwrite the root-level file")
got, readErr := os.ReadFile(filepath.Join(root, "root-secret.txt"))
require.NoError(t, readErr)
assert.Equal(t, rootSecret, string(got), "root-level file must be unchanged")
})
// path.Dir of the joined path can re-introduce a traversal, so check that a
// rejected PUT does not create directories outside the bucket as a side
// effect of making the object's parent directory.
t.Run("PutNoSideEffectDir", func(t *testing.T) {
b, root := newTestBackend(t)
_, err := b.PutObject(ctx, "bucket", "../sneaky-dir/f.txt", map[string]string{}, strings.NewReader("x"), 1)
assert.Error(t, err, "PUT with dot-dot key must be rejected")
_, statErr := os.Stat(filepath.Join(root, "sneaky-dir"))
assert.True(t, os.IsNotExist(statErr), "no directory must be created outside the bucket")
})
// Multipart uploads stream straight to the backing Fs, bypassing the VFS,
// so the key must be rejected before any directory or stream is created.
t.Run("Multipart", func(t *testing.T) {
b, root := newTestBackend(t)
_, err := b.CreateMultipartUpload(ctx, "bucket", "../mp-secret.txt", map[string]string{})
assert.Error(t, err, "multipart upload with dot-dot key must be rejected")
entries, readErr := os.ReadDir(root)
require.NoError(t, readErr)
var names []string
for _, e := range entries {
names = append(names, e.Name())
}
assert.ElementsMatch(t, []string{"bucket", "root-secret.txt"}, names, "no extra entries must be created at the serve root")
})
// A legitimate object inside the bucket must still be readable.
t.Run("LegitimateObject", func(t *testing.T) {
b, _ := newTestBackend(t)
obj, err := b.GetObject(ctx, "bucket", "object.txt", nil)
require.NoError(t, err)
defer func() { _ = obj.Contents.Close() }()
contents, err := io.ReadAll(obj.Contents)
require.NoError(t, err)
assert.Equal(t, "normal object", string(contents))
})
// A legitimate nested object must still be writable, creating its parent
// directory inside the bucket.
t.Run("LegitimateNestedPut", func(t *testing.T) {
b, root := newTestBackend(t)
const payload = "nested"
_, err := b.PutObject(ctx, "bucket", "sub/dir/file.txt", map[string]string{}, strings.NewReader(payload), int64(len(payload)))
require.NoError(t, err)
got, readErr := os.ReadFile(filepath.Join(root, "bucket", "sub", "dir", "file.txt"))
require.NoError(t, readErr)
assert.Equal(t, payload, string(got))
})
}
// TestBucketObjectPath unit-tests the bucket/key join used by serve s3. Object
// keys are opaque, so non-canonical keys (containing "..", ".", "//", or
// leading/trailing slashes) are rejected rather than normalised.
func TestBucketObjectPath(t *testing.T) {
for _, test := range []struct {
bucket, key string
want string
wantErr bool
}{
{"bucket", "object.txt", "bucket/object.txt", false},
{"bucket", "a/b/c.txt", "bucket/a/b/c.txt", false},
{"bucket", "a/../b.txt", "", true}, // distinct from b.txt, not normalised
{"bucket", "a/./b.txt", "", true},
{"bucket", "a//b.txt", "", true},
{"bucket", "/leading", "", true},
{"bucket", "trailing/", "", true},
{"bucket", "", "", true},
{"bucket", "../root-secret.txt", "", true},
{"bucket", "../../etc/passwd", "", true},
{"bucket", "../otherbucket/x", "", true},
{"bucket", "a/../../escape", "", true},
{"bucket", "..", "", true},
} {
got, err := bucketObjectPath(test.bucket, test.key)
if test.wantErr {
assert.Error(t, err, "bucket=%q key=%q", test.bucket, test.key)
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "want 400 InvalidArgument for key=%q, got %v", test.key, err)
} else {
require.NoError(t, err, "bucket=%q key=%q", test.bucket, test.key)
assert.Equal(t, test.want, got, "bucket=%q key=%q", test.bucket, test.key)
}
}
}
// TestBucketDirPath unit-tests the directory-prefix join: the empty prefix
// addresses the bucket root and a trailing slash is ignored, but traversal and
// other non-canonical prefixes are still rejected.
func TestBucketDirPath(t *testing.T) {
for _, test := range []struct {
bucket, dir string
want string
wantErr bool
}{
{"bucket", "", "bucket", false}, // empty prefix addresses the bucket root
{"bucket", "dir", "bucket/dir", false},
{"bucket", "dir/", "", true}, // trailing slash not normalised away
{"bucket", "a/b", "bucket/a/b", false},
{"bucket", "../x", "", true},
{"bucket", "a//b", "", true},
{"bucket", "..", "", true},
} {
got, err := bucketDirPath(test.bucket, test.dir)
if test.wantErr {
assert.Error(t, err, "bucket=%q dir=%q", test.bucket, test.dir)
} else {
require.NoError(t, err, "bucket=%q dir=%q", test.bucket, test.dir)
assert.Equal(t, test.want, got, "bucket=%q dir=%q", test.bucket, test.dir)
}
}
}
// TestEtagHashAuto checks that --etag-hash auto uses the best hash of
// the backend of each auth proxy user rather than of the remote the
// server was started with, which may have none or not exist at all.
func TestEtagHashAuto(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
opt := Opt
opt.EtagHash = "auto"
opt.HTTP.ListenAddr = []string{endpoint}
proxyOpt := proxy.Opt
proxyOpt.AuthProxy = "/path/to/auth/proxy"
fCrypt, err := fs.NewFs(ctx, ":crypt,remote='"+t.TempDir()+"',password="+obscure.MustObscure("password")+":")
require.NoError(t, err)
require.Equal(t, hash.None, fCrypt.Hashes().GetOne(), "crypt has no hashes")
for _, f := range []fs.Fs{nil, fCrypt} {
w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxyOpt)
require.NoError(t, err)
t.Cleanup(func() { _ = w.Shutdown() })
fUser, err := fs.NewFs(ctx, t.TempDir())
require.NoError(t, err)
userVFS := vfs.New(ctx, fUser, &vfscommon.Opt)
t.Cleanup(userVFS.Shutdown)
ctx := context.WithValue(ctx, ctxKeyID, userVFS)
require.NoError(t, w.backend.CreateBucket(ctx, "bucket"))
_, err = w.backend.PutObject(ctx, "bucket", "object", nil, strings.NewReader("data"), 4)
require.NoError(t, err)
obj, err := w.backend.HeadObject(ctx, "bucket", "object")
require.NoError(t, err)
want := md5.Sum([]byte("data"))
assert.Equal(t, want[:], obj.Hash)
}
}
// TestCopyObjectMissingSource checks that copying an object which
// doesn't exist fails with NoSuchKey, including a copy onto itself to
// replace its metadata.
func TestCopyObjectMissingSource(t *testing.T) {
b, _ := newTestBackend(t)
ctx := context.Background()
for _, dstKey := range []string{"missing.txt", "other.txt"} {
_, err := b.CopyObject(ctx, "bucket", "missing.txt", "bucket", dstKey, map[string]string{"X-Amz-Meta-Colour": "red"})
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "copy to %q: %v", dstKey, err)
}
_, err := b.HeadObject(ctx, "bucket", "other.txt")
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey))
}
// tenantCtx returns the context of a request authenticated by the auth
// proxy as accessKeyID.
func tenantCtx(accessKeyID string) context.Context {
return context.WithValue(context.Background(), ctxKeyAccessKeyID, accessKeyID)
}
// TestMetadataOtherTenant checks that with an auth proxy the metadata
// one user (access key ID) stores for an object isn't returned for
// another user's object at the same path.
func TestMetadataOtherTenant(t *testing.T) {
b, _ := newTestBackend(t)
ctxA, ctxB := tenantCtx("tenantA"), tenantCtx("tenantB")
const secretKey = "X-Amz-Meta-Secret"
put := func(ctx context.Context, meta map[string]string) {
_, err := b.PutObject(ctx, "bucket", "meta.txt", meta, strings.NewReader("data"), 4)
require.NoError(t, err)
}
head := func(ctx context.Context) map[string]string {
obj, err := b.HeadObject(ctx, "bucket", "meta.txt")
require.NoError(t, err)
return obj.Metadata
}
put(ctxB, map[string]string{})
put(ctxA, map[string]string{secretKey: "tenantA's"})
assert.Equal(t, "tenantA's", head(ctxA)[secretKey])
assert.NotContains(t, head(ctxB), secretKey)
obj, err := b.GetObject(ctxB, "bucket", "meta.txt", nil)
require.NoError(t, err)
require.NoError(t, obj.Contents.Close())
assert.NotContains(t, obj.Metadata, secretKey)
}
// TestMetadataStale checks that the metadata stored for an object isn't
// returned once the object has been changed by someone else: another
// auth proxy user sharing the backend or a change to the backend itself.
func TestMetadataStale(t *testing.T) {
b, root := newTestBackend(t)
ctxA, ctxB := tenantCtx("tenantA"), tenantCtx("tenantB")
const key = "X-Amz-Meta-Colour"
objPath := filepath.Join(root, "bucket", "meta.txt")
put := func(ctx context.Context, meta map[string]string, data string) {
_, err := b.PutObject(ctx, "bucket", "meta.txt", meta, strings.NewReader(data), int64(len(data)))
require.NoError(t, err)
}
colour := func(ctx context.Context) string {
_vfs, err := b.s.getVFS(ctx)
require.NoError(t, err)
b.forgetPath(_vfs, "bucket/meta.txt")
obj, err := b.HeadObject(ctx, "bucket", "meta.txt")
require.NoError(t, err)
return obj.Metadata[key]
}
put(ctxA, map[string]string{key: "red"}, "data")
assert.Equal(t, "red", colour(ctxA))
put(ctxB, map[string]string{}, "other data")
assert.Equal(t, "", colour(ctxA), "metadata kept after another user replaced the object")
put(ctxA, map[string]string{key: "blue"}, "data")
assert.Equal(t, "blue", colour(ctxA))
later := time.Now().Add(time.Hour)
require.NoError(t, os.Chtimes(objPath, later, later))
assert.Equal(t, "", colour(ctxA), "metadata kept after the object was changed on the backend")
}
// TestCopyObjectMetadata checks a copy gets exactly the metadata given,
// as worked out by gofakes3 from the metadata directive, and keeps the
// source's modification time without adding it to the metadata.
func TestCopyObjectMetadata(t *testing.T) {
b, root := newTestBackend(t)
ctx := context.Background()
srcTime := time.Date(2011, 12, 25, 12, 59, 59, 0, time.UTC)
require.NoError(t, os.Chtimes(filepath.Join(root, "bucket", "object.txt"), srcTime, srcTime))
_, err := b.CopyObject(ctx, "bucket", "object.txt", "bucket", "copy.txt", map[string]string{"X-Amz-Meta-Colour": "red"})
require.NoError(t, err)
obj, err := b.HeadObject(ctx, "bucket", "copy.txt")
require.NoError(t, err)
assert.Equal(t, "red", obj.Metadata["X-Amz-Meta-Colour"])
assert.NotContains(t, obj.Metadata, "X-Amz-Meta-Mtime")
assert.NotContains(t, obj.Metadata, "mtime")
info, err := os.Stat(filepath.Join(root, "bucket", "copy.txt"))
require.NoError(t, err)
assert.True(t, srcTime.Equal(info.ModTime()), "modification time %v", info.ModTime())
// Copying again with other metadata replaces it all
_, err = b.CopyObject(ctx, "bucket", "copy.txt", "bucket", "copy2.txt", map[string]string{"X-Amz-Meta-Size": "big"})
require.NoError(t, err)
obj, err = b.HeadObject(ctx, "bucket", "copy2.txt")
require.NoError(t, err)
assert.Equal(t, "big", obj.Metadata["X-Amz-Meta-Size"])
assert.NotContains(t, obj.Metadata, "X-Amz-Meta-Colour")
}
// TestSetObjectTagging checks the tags of an object can be changed
// without changing its other metadata.
func TestSetObjectTagging(t *testing.T) {
b, _ := newTestBackend(t)
ctx := context.Background()
_, err := b.PutObject(ctx, "bucket", "tagged.txt", map[string]string{"X-Amz-Meta-Colour": "red"}, strings.NewReader("data"), 4)
require.NoError(t, err)
meta := func() map[string]string {
obj, err := b.HeadObject(ctx, "bucket", "tagged.txt")
require.NoError(t, err)
return obj.Metadata
}
require.NoError(t, b.SetObjectTagging(ctx, "bucket", "tagged.txt", "a=1&b=2"))
assert.Equal(t, "a=1&b=2", meta()[gofakes3.TaggingMetadataKey])
assert.Equal(t, "red", meta()["X-Amz-Meta-Colour"])
require.NoError(t, b.SetObjectTagging(ctx, "bucket", "tagged.txt", ""))
assert.NotContains(t, meta(), gofakes3.TaggingMetadataKey)
assert.Equal(t, "red", meta()["X-Amz-Meta-Colour"])
// An object without stored metadata can be tagged
require.NoError(t, b.SetObjectTagging(ctx, "bucket", "object.txt", "c=3"))
obj, err := b.HeadObject(ctx, "bucket", "object.txt")
require.NoError(t, err)
assert.Equal(t, "c=3", obj.Metadata[gofakes3.TaggingMetadataKey])
err = b.SetObjectTagging(ctx, "bucket", "missing.txt", "a=1")
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%v", err)
}
// TestDirMarkers checks directory marker keys, which end in "/", stand
// for directories.
func TestDirMarkers(t *testing.T) {
b, root := newTestBackend(t)
ctx := context.Background()
put := func(key, data string) error {
_, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader(data), int64(len(data)))
return err
}
isDir := func(dir string) bool {
info, err := os.Stat(filepath.Join(root, "bucket", dir))
return err == nil && info.IsDir()
}
// Putting an empty marker makes the directory
require.NoError(t, put("dir/sub/", ""))
assert.True(t, isDir("dir/sub"))
obj, err := b.HeadObject(ctx, "bucket", "dir/sub/")
require.NoError(t, err)
assert.Equal(t, int64(0), obj.Size)
assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", hex.EncodeToString(obj.Hash)) // md5("")
obj, err = b.GetObject(ctx, "bucket", "dir/", nil)
require.NoError(t, err)
data, err := io.ReadAll(obj.Contents)
require.NoError(t, err)
assert.Empty(t, data)
// A marker with data is refused
err = put("data/", "x")
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "%v", err)
assert.False(t, isDir("data"))
// A marker for a directory which doesn't exist isn't found, nor is
// a directory without the "/"
for _, key := range []string{"missing/", "dir"} {
_, err = b.HeadObject(ctx, "bucket", key)
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%s: %v", key, err)
}
// Copying a marker makes the directory of another marker, or an
// empty object
_, err = b.CopyObject(ctx, "bucket", "dir/sub/", "bucket", "copy/", map[string]string{})
require.NoError(t, err)
assert.True(t, isDir("copy"))
_, err = b.CopyObject(ctx, "bucket", "dir/sub/", "bucket", "empty.txt", map[string]string{})
require.NoError(t, err)
obj, err = b.HeadObject(ctx, "bucket", "empty.txt")
require.NoError(t, err)
assert.Equal(t, int64(0), obj.Size)
_, err = b.CopyObject(ctx, "bucket", "dir/", "bucket", "dir/", map[string]string{})
require.NoError(t, err)
// A copy needs the "/" for a directory and none for an object
for _, key := range []string{"dir", "empty.txt/", "missing/"} {
_, err = b.CopyObject(ctx, "bucket", key, "bucket", "other.txt", map[string]string{})
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%s: %v", key, err)
}
// Deleting a marker removes its directory only if empty
require.NoError(t, put("dir/sub/file.txt", "data"))
require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/"))
assert.True(t, isDir("dir/sub"))
require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/file.txt"))
require.NoError(t, put("dir/sub/", ""))
require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/"))
assert.False(t, isDir("dir/sub"))
// Markers must be in canonical form apart from the "/"
for _, key := range []string{"/", "a//", "a/./"} {
err = put(key, "")
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "%s: %v", key, err)
}
}
// TestFileDirConflicts checks keys which conflict with directories or
// files are refused with a client error rather than an internal one,
// which clients would retry.
func TestFileDirConflicts(t *testing.T) {
b, _ := newTestBackend(t)
ctx := context.Background()
put := func(key string) error {
_, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader("data"), 4)
return err
}
require.NoError(t, put("dir/file.txt"))
for _, test := range []struct {
key string
call func(key string) error
}{
{"object.txt/below", put},
{"object.txt/", func(key string) error {
_, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader(""), 0)
return err
}},
{"dir", put},
{"dir", func(key string) error {
_, err := b.CreateMultipartUpload(ctx, "bucket", key, nil)
return err
}},
{"object.txt/below", func(key string) error {
_, err := b.CreateMultipartUpload(ctx, "bucket", key, nil)
return err
}},
} {
err := test.call(test.key)
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidRequest), "%s: %v", test.key, err)
}
}