Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

18.3 KB
Raw Blame
package s3

import (
	"context"
	"crypto/md5"
	"encoding/hex"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/rclone/gofakes3"
	_ "github.com/rclone/rclone/backend/crypt"
	_ "github.com/rclone/rclone/backend/local"
	"github.com/rclone/rclone/cmd/serve/proxy"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/config/obscure"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/fstest"
	"github.com/rclone/rclone/vfs"
	"github.com/rclone/rclone/vfs/vfscommon"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// newTestBackend serves a root directory containing a bucket directory, an
// object inside that bucket and a root-level file that is not part of any
// bucket. It returns the backend and the serve root path.
func newTestBackend(t *testing.T) (*s3Backend, string) {
	fstest.Initialise()
	ctx := context.Background()
	root := t.TempDir()
	require.NoError(t, os.MkdirAll(filepath.Join(root, "bucket"), 0777))
	require.NoError(t, os.WriteFile(filepath.Join(root, "root-secret.txt"), []byte(rootSecret), 0666))
	require.NoError(t, os.WriteFile(filepath.Join(root, "bucket", "object.txt"), []byte("normal object"), 0666))

	f, err := fs.NewFs(ctx, root)
	require.NoError(t, err)

	opt := Opt
	opt.HTTP.ListenAddr = []string{endpoint}
	w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)
	require.NoError(t, err)

	return newBackend(w), root
}

const rootSecret = "ROOT_LEVEL_SECRET_MARKER"

// TestPathTraversal checks that dot-dot segments in an object key cannot
// escape the bucket namespace to read, overwrite or create files outside the
// selected bucket.
func TestPathTraversal(t *testing.T) {
	ctx := context.Background()

	t.Run("Get", func(t *testing.T) {
		b, _ := newTestBackend(t)
		_, err := b.GetObject(ctx, "bucket", "../root-secret.txt", nil)
		assert.Error(t, err, "GET with dot-dot key must not read the root-level file")
	})

	t.Run("Head", func(t *testing.T) {
		b, _ := newTestBackend(t)
		_, err := b.HeadObject(ctx, "bucket", "../root-secret.txt")
		assert.Error(t, err, "HEAD with dot-dot key must not find the root-level file")
	})

	t.Run("Put", func(t *testing.T) {
		b, root := newTestBackend(t)
		const payload = "OVERWRITTEN_BY_DOTDOT"
		_, err := b.PutObject(ctx, "bucket", "../root-secret.txt", map[string]string{}, strings.NewReader(payload), int64(len(payload)))
		assert.Error(t, err, "PUT with dot-dot key must not overwrite the root-level file")
		got, readErr := os.ReadFile(filepath.Join(root, "root-secret.txt"))
		require.NoError(t, readErr)
		assert.Equal(t, rootSecret, string(got), "root-level file must be unchanged")
	})

	// path.Dir of the joined path can re-introduce a traversal, so check that a
	// rejected PUT does not create directories outside the bucket as a side
	// effect of making the object's parent directory.
	t.Run("PutNoSideEffectDir", func(t *testing.T) {
		b, root := newTestBackend(t)
		_, err := b.PutObject(ctx, "bucket", "../sneaky-dir/f.txt", map[string]string{}, strings.NewReader("x"), 1)
		assert.Error(t, err, "PUT with dot-dot key must be rejected")
		_, statErr := os.Stat(filepath.Join(root, "sneaky-dir"))
		assert.True(t, os.IsNotExist(statErr), "no directory must be created outside the bucket")
	})

	// Multipart uploads stream straight to the backing Fs, bypassing the VFS,
	// so the key must be rejected before any directory or stream is created.
	t.Run("Multipart", func(t *testing.T) {
		b, root := newTestBackend(t)
		_, err := b.CreateMultipartUpload(ctx, "bucket", "../mp-secret.txt", map[string]string{})
		assert.Error(t, err, "multipart upload with dot-dot key must be rejected")
		entries, readErr := os.ReadDir(root)
		require.NoError(t, readErr)
		var names []string
		for _, e := range entries {
			names = append(names, e.Name())
		}
		assert.ElementsMatch(t, []string{"bucket", "root-secret.txt"}, names, "no extra entries must be created at the serve root")
	})

	// A legitimate object inside the bucket must still be readable.
	t.Run("LegitimateObject", func(t *testing.T) {
		b, _ := newTestBackend(t)
		obj, err := b.GetObject(ctx, "bucket", "object.txt", nil)
		require.NoError(t, err)
		defer func() { _ = obj.Contents.Close() }()
		contents, err := io.ReadAll(obj.Contents)
		require.NoError(t, err)
		assert.Equal(t, "normal object", string(contents))
	})

	// A legitimate nested object must still be writable, creating its parent
	// directory inside the bucket.
	t.Run("LegitimateNestedPut", func(t *testing.T) {
		b, root := newTestBackend(t)
		const payload = "nested"
		_, err := b.PutObject(ctx, "bucket", "sub/dir/file.txt", map[string]string{}, strings.NewReader(payload), int64(len(payload)))
		require.NoError(t, err)
		got, readErr := os.ReadFile(filepath.Join(root, "bucket", "sub", "dir", "file.txt"))
		require.NoError(t, readErr)
		assert.Equal(t, payload, string(got))
	})
}

// TestBucketObjectPath unit-tests the bucket/key join used by serve s3. Object
// keys are opaque, so non-canonical keys (containing "..", ".", "//", or
// leading/trailing slashes) are rejected rather than normalised.
func TestBucketObjectPath(t *testing.T) {
	for _, test := range []struct {
		bucket, key string
		want        string
		wantErr     bool
	}{
		{"bucket", "object.txt", "bucket/object.txt", false},
		{"bucket", "a/b/c.txt", "bucket/a/b/c.txt", false},
		{"bucket", "a/../b.txt", "", true}, // distinct from b.txt, not normalised
		{"bucket", "a/./b.txt", "", true},
		{"bucket", "a//b.txt", "", true},
		{"bucket", "/leading", "", true},
		{"bucket", "trailing/", "", true},
		{"bucket", "", "", true},
		{"bucket", "../root-secret.txt", "", true},
		{"bucket", "../../etc/passwd", "", true},
		{"bucket", "../otherbucket/x", "", true},
		{"bucket", "a/../../escape", "", true},
		{"bucket", "..", "", true},
	} {
		got, err := bucketObjectPath(test.bucket, test.key)
		if test.wantErr {
			assert.Error(t, err, "bucket=%q key=%q", test.bucket, test.key)
			assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "want 400 InvalidArgument for key=%q, got %v", test.key, err)
		} else {
			require.NoError(t, err, "bucket=%q key=%q", test.bucket, test.key)
			assert.Equal(t, test.want, got, "bucket=%q key=%q", test.bucket, test.key)
		}
	}
}

// TestBucketDirPath unit-tests the directory-prefix join: the empty prefix
// addresses the bucket root and a trailing slash is ignored, but traversal and
// other non-canonical prefixes are still rejected.
func TestBucketDirPath(t *testing.T) {
	for _, test := range []struct {
		bucket, dir string
		want        string
		wantErr     bool
	}{
		{"bucket", "", "bucket", false}, // empty prefix addresses the bucket root
		{"bucket", "dir", "bucket/dir", false},
		{"bucket", "dir/", "", true}, // trailing slash not normalised away
		{"bucket", "a/b", "bucket/a/b", false},
		{"bucket", "../x", "", true},
		{"bucket", "a//b", "", true},
		{"bucket", "..", "", true},
	} {
		got, err := bucketDirPath(test.bucket, test.dir)
		if test.wantErr {
			assert.Error(t, err, "bucket=%q dir=%q", test.bucket, test.dir)
		} else {
			require.NoError(t, err, "bucket=%q dir=%q", test.bucket, test.dir)
			assert.Equal(t, test.want, got, "bucket=%q dir=%q", test.bucket, test.dir)
		}
	}
}

// TestEtagHashAuto checks that --etag-hash auto uses the best hash of
// the backend of each auth proxy user rather than of the remote the
// server was started with, which may have none or not exist at all.
func TestEtagHashAuto(t *testing.T) {
	fstest.Initialise()
	ctx := context.Background()
	opt := Opt
	opt.EtagHash = "auto"
	opt.HTTP.ListenAddr = []string{endpoint}
	proxyOpt := proxy.Opt
	proxyOpt.AuthProxy = "/path/to/auth/proxy"

	fCrypt, err := fs.NewFs(ctx, ":crypt,remote='"+t.TempDir()+"',password="+obscure.MustObscure("password")+":")
	require.NoError(t, err)
	require.Equal(t, hash.None, fCrypt.Hashes().GetOne(), "crypt has no hashes")
	for _, f := range []fs.Fs{nil, fCrypt} {
		w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxyOpt)
		require.NoError(t, err)
		t.Cleanup(func() { _ = w.Shutdown() })

		fUser, err := fs.NewFs(ctx, t.TempDir())
		require.NoError(t, err)
		userVFS := vfs.New(ctx, fUser, &vfscommon.Opt)
		t.Cleanup(userVFS.Shutdown)
		ctx := context.WithValue(ctx, ctxKeyID, userVFS)
		require.NoError(t, w.backend.CreateBucket(ctx, "bucket"))
		_, err = w.backend.PutObject(ctx, "bucket", "object", nil, strings.NewReader("data"), 4)
		require.NoError(t, err)

		obj, err := w.backend.HeadObject(ctx, "bucket", "object")
		require.NoError(t, err)
		want := md5.Sum([]byte("data"))
		assert.Equal(t, want[:], obj.Hash)
	}
}

// TestCopyObjectMissingSource checks that copying an object which
// doesn't exist fails with NoSuchKey, including a copy onto itself to
// replace its metadata.
func TestCopyObjectMissingSource(t *testing.T) {
	b, _ := newTestBackend(t)
	ctx := context.Background()
	for _, dstKey := range []string{"missing.txt", "other.txt"} {
		_, err := b.CopyObject(ctx, "bucket", "missing.txt", "bucket", dstKey, map[string]string{"X-Amz-Meta-Colour": "red"})
		assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "copy to %q: %v", dstKey, err)
	}
	_, err := b.HeadObject(ctx, "bucket", "other.txt")
	assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey))
}

// tenantCtx returns the context of a request authenticated by the auth
// proxy as accessKeyID.
func tenantCtx(accessKeyID string) context.Context {
	return context.WithValue(context.Background(), ctxKeyAccessKeyID, accessKeyID)
}

// TestMetadataOtherTenant checks that with an auth proxy the metadata
// one user (access key ID) stores for an object isn't returned for
// another user's object at the same path.
func TestMetadataOtherTenant(t *testing.T) {
	b, _ := newTestBackend(t)
	ctxA, ctxB := tenantCtx("tenantA"), tenantCtx("tenantB")
	const secretKey = "X-Amz-Meta-Secret"

	put := func(ctx context.Context, meta map[string]string) {
		_, err := b.PutObject(ctx, "bucket", "meta.txt", meta, strings.NewReader("data"), 4)
		require.NoError(t, err)
	}
	head := func(ctx context.Context) map[string]string {
		obj, err := b.HeadObject(ctx, "bucket", "meta.txt")
		require.NoError(t, err)
		return obj.Metadata
	}

	put(ctxB, map[string]string{})
	put(ctxA, map[string]string{secretKey: "tenantA's"})

	assert.Equal(t, "tenantA's", head(ctxA)[secretKey])
	assert.NotContains(t, head(ctxB), secretKey)

	obj, err := b.GetObject(ctxB, "bucket", "meta.txt", nil)
	require.NoError(t, err)
	require.NoError(t, obj.Contents.Close())
	assert.NotContains(t, obj.Metadata, secretKey)
}

// TestMetadataStale checks that the metadata stored for an object isn't
// returned once the object has been changed by someone else: another
// auth proxy user sharing the backend or a change to the backend itself.
func TestMetadataStale(t *testing.T) {
	b, root := newTestBackend(t)
	ctxA, ctxB := tenantCtx("tenantA"), tenantCtx("tenantB")
	const key = "X-Amz-Meta-Colour"
	objPath := filepath.Join(root, "bucket", "meta.txt")

	put := func(ctx context.Context, meta map[string]string, data string) {
		_, err := b.PutObject(ctx, "bucket", "meta.txt", meta, strings.NewReader(data), int64(len(data)))
		require.NoError(t, err)
	}
	colour := func(ctx context.Context) string {
		_vfs, err := b.s.getVFS(ctx)
		require.NoError(t, err)
		b.forgetPath(_vfs, "bucket/meta.txt")
		obj, err := b.HeadObject(ctx, "bucket", "meta.txt")
		require.NoError(t, err)
		return obj.Metadata[key]
	}

	put(ctxA, map[string]string{key: "red"}, "data")
	assert.Equal(t, "red", colour(ctxA))

	put(ctxB, map[string]string{}, "other data")
	assert.Equal(t, "", colour(ctxA), "metadata kept after another user replaced the object")

	put(ctxA, map[string]string{key: "blue"}, "data")
	assert.Equal(t, "blue", colour(ctxA))
	later := time.Now().Add(time.Hour)
	require.NoError(t, os.Chtimes(objPath, later, later))
	assert.Equal(t, "", colour(ctxA), "metadata kept after the object was changed on the backend")
}

// TestCopyObjectMetadata checks a copy gets exactly the metadata given,
// as worked out by gofakes3 from the metadata directive, and keeps the
// source's modification time without adding it to the metadata.
func TestCopyObjectMetadata(t *testing.T) {
	b, root := newTestBackend(t)
	ctx := context.Background()
	srcTime := time.Date(2011, 12, 25, 12, 59, 59, 0, time.UTC)
	require.NoError(t, os.Chtimes(filepath.Join(root, "bucket", "object.txt"), srcTime, srcTime))

	_, err := b.CopyObject(ctx, "bucket", "object.txt", "bucket", "copy.txt", map[string]string{"X-Amz-Meta-Colour": "red"})
	require.NoError(t, err)
	obj, err := b.HeadObject(ctx, "bucket", "copy.txt")
	require.NoError(t, err)
	assert.Equal(t, "red", obj.Metadata["X-Amz-Meta-Colour"])
	assert.NotContains(t, obj.Metadata, "X-Amz-Meta-Mtime")
	assert.NotContains(t, obj.Metadata, "mtime")
	info, err := os.Stat(filepath.Join(root, "bucket", "copy.txt"))
	require.NoError(t, err)
	assert.True(t, srcTime.Equal(info.ModTime()), "modification time %v", info.ModTime())

	// Copying again with other metadata replaces it all
	_, err = b.CopyObject(ctx, "bucket", "copy.txt", "bucket", "copy2.txt", map[string]string{"X-Amz-Meta-Size": "big"})
	require.NoError(t, err)
	obj, err = b.HeadObject(ctx, "bucket", "copy2.txt")
	require.NoError(t, err)
	assert.Equal(t, "big", obj.Metadata["X-Amz-Meta-Size"])
	assert.NotContains(t, obj.Metadata, "X-Amz-Meta-Colour")
}

// TestSetObjectTagging checks the tags of an object can be changed
// without changing its other metadata.
func TestSetObjectTagging(t *testing.T) {
	b, _ := newTestBackend(t)
	ctx := context.Background()
	_, err := b.PutObject(ctx, "bucket", "tagged.txt", map[string]string{"X-Amz-Meta-Colour": "red"}, strings.NewReader("data"), 4)
	require.NoError(t, err)
	meta := func() map[string]string {
		obj, err := b.HeadObject(ctx, "bucket", "tagged.txt")
		require.NoError(t, err)
		return obj.Metadata
	}

	require.NoError(t, b.SetObjectTagging(ctx, "bucket", "tagged.txt", "a=1&b=2"))
	assert.Equal(t, "a=1&b=2", meta()[gofakes3.TaggingMetadataKey])
	assert.Equal(t, "red", meta()["X-Amz-Meta-Colour"])

	require.NoError(t, b.SetObjectTagging(ctx, "bucket", "tagged.txt", ""))
	assert.NotContains(t, meta(), gofakes3.TaggingMetadataKey)
	assert.Equal(t, "red", meta()["X-Amz-Meta-Colour"])

	// An object without stored metadata can be tagged
	require.NoError(t, b.SetObjectTagging(ctx, "bucket", "object.txt", "c=3"))
	obj, err := b.HeadObject(ctx, "bucket", "object.txt")
	require.NoError(t, err)
	assert.Equal(t, "c=3", obj.Metadata[gofakes3.TaggingMetadataKey])

	err = b.SetObjectTagging(ctx, "bucket", "missing.txt", "a=1")
	assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%v", err)
}

// TestDirMarkers checks directory marker keys, which end in "/", stand
// for directories.
func TestDirMarkers(t *testing.T) {
	b, root := newTestBackend(t)
	ctx := context.Background()
	put := func(key, data string) error {
		_, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader(data), int64(len(data)))
		return err
	}
	isDir := func(dir string) bool {
		info, err := os.Stat(filepath.Join(root, "bucket", dir))
		return err == nil && info.IsDir()
	}

	// Putting an empty marker makes the directory
	require.NoError(t, put("dir/sub/", ""))
	assert.True(t, isDir("dir/sub"))
	obj, err := b.HeadObject(ctx, "bucket", "dir/sub/")
	require.NoError(t, err)
	assert.Equal(t, int64(0), obj.Size)
	assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", hex.EncodeToString(obj.Hash)) // md5("")
	obj, err = b.GetObject(ctx, "bucket", "dir/", nil)
	require.NoError(t, err)
	data, err := io.ReadAll(obj.Contents)
	require.NoError(t, err)
	assert.Empty(t, data)

	// A marker with data is refused
	err = put("data/", "x")
	assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "%v", err)
	assert.False(t, isDir("data"))

	// A marker for a directory which doesn't exist isn't found, nor is
	// a directory without the "/"
	for _, key := range []string{"missing/", "dir"} {
		_, err = b.HeadObject(ctx, "bucket", key)
		assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%s: %v", key, err)
	}

	// Copying a marker makes the directory of another marker, or an
	// empty object
	_, err = b.CopyObject(ctx, "bucket", "dir/sub/", "bucket", "copy/", map[string]string{})
	require.NoError(t, err)
	assert.True(t, isDir("copy"))
	_, err = b.CopyObject(ctx, "bucket", "dir/sub/", "bucket", "empty.txt", map[string]string{})
	require.NoError(t, err)
	obj, err = b.HeadObject(ctx, "bucket", "empty.txt")
	require.NoError(t, err)
	assert.Equal(t, int64(0), obj.Size)
	_, err = b.CopyObject(ctx, "bucket", "dir/", "bucket", "dir/", map[string]string{})
	require.NoError(t, err)

	// A copy needs the "/" for a directory and none for an object
	for _, key := range []string{"dir", "empty.txt/", "missing/"} {
		_, err = b.CopyObject(ctx, "bucket", key, "bucket", "other.txt", map[string]string{})
		assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%s: %v", key, err)
	}

	// Deleting a marker removes its directory only if empty
	require.NoError(t, put("dir/sub/file.txt", "data"))
	require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/"))
	assert.True(t, isDir("dir/sub"))
	require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/file.txt"))
	require.NoError(t, put("dir/sub/", ""))
	require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/"))
	assert.False(t, isDir("dir/sub"))

	// Markers must be in canonical form apart from the "/"
	for _, key := range []string{"/", "a//", "a/./"} {
		err = put(key, "")
		assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "%s: %v", key, err)
	}
}

// TestFileDirConflicts checks keys which conflict with directories or
// files are refused with a client error rather than an internal one,
// which clients would retry.
func TestFileDirConflicts(t *testing.T) {
	b, _ := newTestBackend(t)
	ctx := context.Background()
	put := func(key string) error {
		_, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader("data"), 4)
		return err
	}
	require.NoError(t, put("dir/file.txt"))

	for _, test := range []struct {
		key  string
		call func(key string) error
	}{
		{"object.txt/below", put},
		{"object.txt/", func(key string) error {
			_, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader(""), 0)
			return err
		}},
		{"dir", put},
		{"dir", func(key string) error {
			_, err := b.CreateMultipartUpload(ctx, "bucket", key, nil)
			return err
		}},
		{"object.txt/below", func(key string) error {
			_, err := b.CreateMultipartUpload(ctx, "bucket", key, nil)
			return err
		}},
	} {
		err := test.call(test.key)
		assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidRequest), "%s: %v", test.key, err)
	}
}