"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
package s3
import (
"context"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"github.com/rclone/gofakes3"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// newListBackend serves a temporary directory populated with the given files,
// whose paths are relative to the serve root and so start with a bucket name.
func newListBackend(t *testing.T, files ...string) *s3Backend {
return newListBackendOpt(t, Opt, files...)
}
// newListBackendOpt is newListBackend with the options opt.
func newListBackendOpt(t *testing.T, opt Options, files ...string) *s3Backend {
fstest.Initialise()
ctx := context.Background()
root := t.TempDir()
for _, file := range files {
p := filepath.Join(root, filepath.FromSlash(file))
require.NoError(t, os.MkdirAll(filepath.Dir(p), 0777))
require.NoError(t, os.WriteFile(p, []byte(file), 0666))
}
f, err := fs.NewFs(ctx, root)
require.NoError(t, err)
opt.HTTP.ListenAddr = []string{endpoint}
w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)
require.NoError(t, err)
// Use the server's own backend so that a test which drives the server
// and one which calls the backend directly share the same state.
return w.backend
}
// listPrefix makes a listing prefix, with a delimiter if delimiter is set.
func listPrefix(prefix, delimiter string) *gofakes3.Prefix {
return &gofakes3.Prefix{
HasPrefix: prefix != "",
Prefix: prefix,
HasDelimiter: delimiter != "",
Delimiter: delimiter,
}
}
// listOnce returns the keys of one page of a listing, common prefixes and
// object keys merged into the single sorted stream S3 clients see.
func listOnce(t *testing.T, b *s3Backend, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (keys []string, list *gofakes3.ObjectList) {
list, err := b.ListBucket(context.Background(), "bucket", prefix, page)
require.NoError(t, err)
for _, commonPrefix := range list.CommonPrefixes {
keys = append(keys, commonPrefix.Prefix)
}
for _, object := range list.Contents {
keys = append(keys, object.Key)
}
slices.Sort(keys)
return keys, list
}
// listPages pages through a whole listing maxKeys at a time, the way an S3
// client does, and returns the keys of every page concatenated.
func listPages(t *testing.T, b *s3Backend, prefix *gofakes3.Prefix, maxKeys int) (keys []string) {
page := gofakes3.ListBucketPage{MaxKeys: int64(maxKeys)}
for pages := 0; ; pages++ {
require.Less(t, pages, 1000, "listing did not terminate")
pageKeys, list := listOnce(t, b, prefix, page)
assert.LessOrEqual(t, len(pageKeys), maxKeys, "page has more than max-keys keys")
keys = append(keys, pageKeys...)
if !list.IsTruncated {
return keys
}
require.NotEmpty(t, list.NextMarker, "truncated page has no next marker")
require.Equal(t, list.NextMarker, pageKeys[len(pageKeys)-1], "next marker is not the last key of the page")
page.Marker, page.HasMarker = list.NextMarker, true
}
}
// TestListFlatKeyOrder checks that keys come back in the order of a flat S3
// keyspace rather than in the order a directory walk finds them: "a.txt"
// sorts before "a/b" because '.' < '/'.
func TestListFlatKeyOrder(t *testing.T) {
b := newListBackend(t, "bucket/a/b", "bucket/a.txt", "bucket/a0")
keys, list := listOnce(t, b, listPrefix("", ""), gofakes3.ListBucketPage{MaxKeys: 10})
assert.Equal(t, []string{"a.txt", "a/b", "a0"}, keys)
assert.False(t, list.IsTruncated)
// Paging must not skip a key across the "a.txt"/"a/b" boundary
assert.Equal(t, []string{"a.txt", "a/b", "a0"}, listPages(t, b, listPrefix("", ""), 1))
}
// TestListPaging checks that paging through a listing returns every key
// exactly once, whatever the page size.
func TestListPaging(t *testing.T) {
var files []string
for dir := range 5 {
for file := range 4 {
files = append(files, fmt.Sprintf("bucket/dir%d/file%d", dir, file))
}
files = append(files, fmt.Sprintf("bucket/dir%d.txt", dir))
}
b := newListBackend(t, files...)
all, list := listOnce(t, b, listPrefix("", ""), gofakes3.ListBucketPage{MaxKeys: 1000})
require.False(t, list.IsTruncated)
require.Len(t, all, 25)
assert.True(t, slices.IsSorted(all), "unpaged listing is not sorted: %q", all)
for _, maxKeys := range []int{1, 2, 3, 7, 24, 25, 26} {
t.Run(fmt.Sprintf("MaxKeys%d", maxKeys), func(t *testing.T) {
assert.Equal(t, all, listPages(t, b, listPrefix("", ""), maxKeys))
})
}
}
// TestListPagingWithPrefix checks paging a listing which is filtered by a
// prefix that is not a whole path segment.
func TestListPagingWithPrefix(t *testing.T) {
b := newListBackend(t,
"bucket/apple/1", "bucket/apple/2", "bucket/apricot/1",
"bucket/apt.txt", "bucket/banana/1",
)
prefix := listPrefix("ap", "")
all, _ := listOnce(t, b, prefix, gofakes3.ListBucketPage{MaxKeys: 1000})
assert.Equal(t, []string{"apple/1", "apple/2", "apricot/1", "apt.txt"}, all)
for _, maxKeys := range []int{1, 2, 3} {
t.Run(fmt.Sprintf("MaxKeys%d", maxKeys), func(t *testing.T) {
assert.Equal(t, all, listPages(t, b, prefix, maxKeys))
})
}
}
// TestListDelimiter checks that a delimited listing pages through common
// prefixes and object keys as one merged stream.
func TestListDelimiter(t *testing.T) {
b := newListBackend(t,
"bucket/a/1", "bucket/b.txt", "bucket/c/1", "bucket/d.txt", "bucket/e/1",
)
prefix := listPrefix("", "/")
all, _ := listOnce(t, b, prefix, gofakes3.ListBucketPage{MaxKeys: 1000})
assert.Equal(t, []string{"a/", "b.txt", "c/", "d.txt", "e/"}, all)
for _, maxKeys := range []int{1, 2, 3, 4} {
t.Run(fmt.Sprintf("MaxKeys%d", maxKeys), func(t *testing.T) {
assert.Equal(t, all, listPages(t, b, prefix, maxKeys))
})
}
}
// countDirs lists one page directly with a lister so that the number of
// directories the walk had to read can be checked.
func countDirs(t *testing.T, b *s3Backend, page gofakes3.ListBucketPage) (dirsRead int, list *gofakes3.ObjectList) {
_vfs, err := b.s.getVFS(context.Background())
require.NoError(t, err)
list = gofakes3.NewObjectList()
l := newLister(context.Background(), b, _vfs, "bucket", page, list)
err = l.list("", "", false)
if err != nil && !errors.Is(err, errPageFull) {
require.NoError(t, err)
}
return l.dirsRead, list
}
// TestListReadsOnlyTheDirectoriesItNeeds checks that a page costs a number of
// directory reads proportional to the keys it returns, not to the size of the
// subtree, and that resuming from a marker does not re-read the subtrees
// earlier pages already returned.
func TestListReadsOnlyTheDirectoriesItNeeds(t *testing.T) {
const dirs = 20
var files []string
for dir := range dirs {
for file := range 5 {
files = append(files, fmt.Sprintf("bucket/dir%02d/file%d", dir, file))
}
}
b := newListBackend(t, files...)
// A full listing has to read the bucket and every directory in it
whole, list := countDirs(t, b, gofakes3.ListBucketPage{MaxKeys: 1000})
require.False(t, list.IsTruncated)
require.Len(t, list.Contents, dirs*5)
assert.Equal(t, dirs+1, whole)
// Each page of 5 reads the bucket, the directory holding the marker,
// the directory it returns and the one holding the following key
page := gofakes3.ListBucketPage{MaxKeys: 5}
for pages := range dirs {
dirsRead, list := countDirs(t, b, page)
assert.LessOrEqual(t, dirsRead, 4, "page %d read too many directories", pages)
assert.Less(t, dirsRead, whole, "page %d walked the whole tree", pages)
require.Len(t, list.Contents, 5)
assert.Equal(t, pages == dirs-1, !list.IsTruncated)
page.Marker, page.HasMarker = list.NextMarker, true
}
}
// TestListHidesTemporaryObjects checks that the temporary objects of
// in-progress uploads stay hidden and do not use up a page.
func TestListHidesTemporaryObjects(t *testing.T) {
b := newListBackend(t,
"bucket/a.txt",
"bucket/"+putObjectPrefix+"hidden",
"bucket/"+legacyMultipartUploadPrefix+"hidden",
"bucket/z.txt",
)
keys, list := listOnce(t, b, listPrefix("", ""), gofakes3.ListBucketPage{MaxKeys: 2})
assert.Equal(t, []string{"a.txt", "z.txt"}, keys)
assert.False(t, list.IsTruncated)
}
// TestListSortKey checks the key a directory entry sorts under.
func TestListSortKey(t *testing.T) {
b := newListBackend(t, "bucket/dir/file", "bucket/file")
_vfs, err := b.s.getVFS(context.Background())
require.NoError(t, err)
entries, err := getDirEntries("bucket", _vfs)
require.NoError(t, err)
got := map[string]string{}
for _, entry := range entries {
got[entry.Name()] = sortKey(entry)
}
assert.Equal(t, map[string]string{"dir": "dir/", "file": "file"}, got)
}
// TestListSkipDir checks which subtrees a marker allows to be skipped unread.
func TestListSkipDir(t *testing.T) {
for _, test := range []struct {
marker string
dir string
want bool
}{
{marker: "", dir: "a/", want: false}, // no marker, list everything
{marker: "a", dir: "b/", want: false}, // marker before the subtree
{marker: "b/9", dir: "b/", want: false}, // marker inside the subtree
{marker: "b/", dir: "b/", want: false}, // marker is the subtree
{marker: "b0", dir: "b/", want: true}, // marker past the subtree
{marker: "c", dir: "b/", want: true}, // marker past the subtree
{marker: "b.txt", dir: "b/", want: false}, // '.' sorts before '/'
{marker: "b/a", dir: "b/c/", want: false}, // marker before the subtree
{marker: "b/c/9", dir: "b/c/", want: false}, // marker inside the subtree
{marker: "b/d", dir: "b/c/", want: true}, // marker past the subtree
} {
l := &lister{marker: test.marker, hasMarker: test.marker != ""}
assert.Equal(t, test.want, l.skipDir(test.dir), "marker %q dir %q", test.marker, test.dir)
}
}
// TestListAnyDelimiter checks listings with any delimiter, and with a
// prefix, against the keys an S3 listing of the same flat keyspace has,
// whatever the page size.
func TestListAnyDelimiter(t *testing.T) {
files := []string{
"a.txt", "a/b/c", "a/bc", "a0",
"b%ar", "b ar", "b.ar", "bar", "bazar", "baz/qux", "baz/q ux/x",
"cab", "c/a/b/z", "c/ab", "dir/sub/file", "foo",
}
var paths []string
for _, file := range files {
paths = append(paths, "bucket/"+file)
}
// A directory holding only the temporary object of an upload has
// no keys
paths = append(paths, "bucket/pending/"+putObjectPrefix+"x")
b := newListBackend(t, paths...)
// An empty directory has its directory marker as its key
require.NoError(t, os.MkdirAll(filepath.Join(b.s.f.Root(), "bucket", "empty", "abc"), 0777))
files = append(files, "empty/abc/")
// want returns the keys and common prefixes S3 lists
want := func(prefix, delimiter string) (keys []string) {
seen := map[string]bool{}
keySpace := files
if delimiter == "/" && prefix != "pending/" {
// Listing directories with "/" shows every directory
// as a common prefix, as it always has
keySpace = append(slices.Clone(files), "pending/")
}
for _, key := range keySpace {
if !strings.HasPrefix(key, prefix) {
continue
}
if delimiter != "" {
if i := strings.Index(key[len(prefix):], delimiter); i >= 0 {
key = key[:len(prefix)+i+len(delimiter)]
}
}
if !seen[key] {
seen[key] = true
keys = append(keys, key)
}
}
slices.Sort(keys)
return keys
}
for _, prefix := range []string{"", "b", "ba", "baz/", "c", "c/a", "e", "empty/", "empty/abc", "empty/abc/", "p", "pending/", "x"} {
for _, delimiter := range []string{"", "/", "a", "ba", " ", ".", "%", "/a", "b/", "z"} {
t.Run(fmt.Sprintf("prefix=%q,delimiter=%q", prefix, delimiter), func(t *testing.T) {
expected := want(prefix, delimiter)
for _, maxKeys := range []int{1, 2, 3, 1000} {
got := listPages(t, b, listPrefix(prefix, delimiter), maxKeys)
assert.Equal(t, expected, got, "max-keys %d", maxKeys)
}
})
}
}
}
// TestListPrefixLeadingSlash checks a prefix starting with "/" matches
// nothing, as no key starts with "/".
func TestListPrefixLeadingSlash(t *testing.T) {
b := newListBackend(t, "bucket/a/b", "bucket/c")
keys, _ := listOnce(t, b, listPrefix("/", ""), gofakes3.ListBucketPage{MaxKeys: 1000})
assert.Empty(t, keys)
keys, _ = listOnce(t, b, listPrefix("/a", "/"), gofakes3.ListBucketPage{MaxKeys: 1000})
assert.Empty(t, keys)
}
// TestListETagQuoted checks the ETags in listings are quoted as on S3
// and in HEAD responses.
func TestListETagQuoted(t *testing.T) {
b := newListBackend(t, "bucket/a")
_, list := listOnce(t, b, listPrefix("", ""), gofakes3.ListBucketPage{MaxKeys: 1000})
require.Len(t, list.Contents, 1)
obj, err := b.HeadObject(context.Background(), "bucket", "a")
require.NoError(t, err)
assert.Equal(t, `"`+hex.EncodeToString(obj.Hash)+`"`, list.Contents[0].ETag)
}
// TestListPutDirMarkers checks that the directory marker put for a
// directory is listed, and kept, while the directory holds other keys,
// as on S3, but not that of a directory made by putting a key below it.
func TestListPutDirMarkers(t *testing.T) {
b := newListBackend(t, "bucket/implicit/file")
ctx := context.Background()
put := func(bucket, key string) {
_, err := b.PutObject(ctx, bucket, key, nil, strings.NewReader(""), 0)
require.NoError(t, err)
}
del := func(key string) {
require.NoError(t, b.deleteObject(ctx, "bucket", key))
}
all := func() []string {
return listPages(t, b, listPrefix("", ""), 1000)
}
put("bucket", "dir/")
put("bucket", "dir/a")
put("bucket", "dir/sub/b")
want := []string{"dir/", "dir/a", "dir/sub/b", "implicit/file"}
assert.Equal(t, want, all())
assert.Equal(t, want, listPages(t, b, listPrefix("", ""), 1))
keys, _ := listOnce(t, b, listPrefix("", "/"), gofakes3.ListBucketPage{})
assert.Equal(t, []string{"dir/", "implicit/"}, keys)
keys, _ = listOnce(t, b, listPrefix("dir/", "/"), gofakes3.ListBucketPage{})
assert.Equal(t, []string{"dir/", "dir/a", "dir/sub/"}, keys)
keys, _ = listOnce(t, b, listPrefix("", "r"), gofakes3.ListBucketPage{})
assert.Equal(t, []string{"dir", "implicit/file"}, keys)
// The marker keeps its directory when the keys below it are deleted
del("dir/a")
del("dir/sub/b")
assert.Equal(t, []string{"dir/", "implicit/file"}, all())
// Deleting the marker leaves the keys below it, and then the
// directory goes with the last of them
put("bucket", "dir/c")
del("dir/")
assert.Equal(t, []string{"dir/c", "implicit/file"}, all())
del("dir/c")
assert.Equal(t, []string{"implicit/file"}, all())
// The markers of a deleted bucket are forgotten
require.NoError(t, b.CreateBucket(ctx, "other"))
put("other", "d/")
require.NoError(t, b.DeleteBucket(ctx, "other"))
require.NoError(t, b.CreateBucket(ctx, "other"))
put("other", "d/f")
list, err := b.ListBucket(ctx, "other", listPrefix("", ""), gofakes3.ListBucketPage{})
require.NoError(t, err)
require.Len(t, list.Contents, 1)
assert.Equal(t, "d/f", list.Contents[0].Key)
}
// TestListStorageClass checks the storage class an object was put with
// is listed.
func TestListStorageClass(t *testing.T) {
b := newListBackend(t, "bucket/existing")
ctx := context.Background()
_, err := b.PutObject(ctx, "bucket", "ia", map[string]string{"X-Amz-Storage-Class": "STANDARD_IA"}, strings.NewReader("ia"), 2)
require.NoError(t, err)
_, err = b.PutObject(ctx, "bucket", "plain", nil, strings.NewReader("plain"), 5)
require.NoError(t, err)
_, list := listOnce(t, b, listPrefix("", ""), gofakes3.ListBucketPage{})
got := map[string]gofakes3.StorageClass{}
for _, c := range list.Contents {
got[c.Key] = c.StorageClass
}
assert.Equal(t, map[string]gofakes3.StorageClass{
"existing": gofakes3.StorageStandard,
"ia": "STANDARD_IA",
"plain": gofakes3.StorageStandard,
}, got)
}
// TestListDirectoryMarkers checks every directory is listed as a
// directory marker with --directory-markers.
func TestListDirectoryMarkers(t *testing.T) {
opt := Opt
opt.DirectoryMarkers = true
b := newListBackendOpt(t, opt, "bucket/implicit/file", "bucket/a/b/c")
ctx := context.Background()
all := func() []string {
return listPages(t, b, listPrefix("", ""), 1000)
}
want := []string{"a/", "a/b/", "a/b/c", "implicit/", "implicit/file"}
assert.Equal(t, want, all())
assert.Equal(t, want, listPages(t, b, listPrefix("", ""), 1))
keys, _ := listOnce(t, b, listPrefix("a/", "/"), gofakes3.ListBucketPage{})
assert.Equal(t, []string{"a/", "a/b/"}, keys)
keys, _ = listOnce(t, b, listPrefix("", "b"), gofakes3.ListBucketPage{})
assert.Equal(t, []string{"a/", "a/b", "implicit/", "implicit/file"}, keys)
// The directories are kept when the keys below them are deleted
require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/c"))
assert.Equal(t, []string{"a/", "a/b/", "implicit/", "implicit/file"}, all())
require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/"))
assert.Equal(t, []string{"a/", "implicit/", "implicit/file"}, all())
// A marker deleted while its directory holds other keys isn't
// listed, and the directory goes with the last key below it, in
// whatever order the keys are deleted
_, err := b.PutObject(ctx, "bucket", "x/y/z", nil, strings.NewReader("z"), 1)
require.NoError(t, err)
assert.Equal(t, []string{"a/", "implicit/", "implicit/file", "x/", "x/y/", "x/y/z"}, all())
require.NoError(t, b.deleteObject(ctx, "bucket", "x/"))
assert.Equal(t, []string{"a/", "implicit/", "implicit/file", "x/y/", "x/y/z"}, all())
require.NoError(t, b.deleteObject(ctx, "bucket", "x/y/z"))
assert.Equal(t, []string{"a/", "implicit/", "implicit/file", "x/y/"}, all())
require.NoError(t, b.deleteObject(ctx, "bucket", "x/y/"))
assert.Equal(t, []string{"a/", "implicit/", "implicit/file"}, all())
// A directory made again has its marker
_, err = b.PutObject(ctx, "bucket", "x/w", nil, strings.NewReader("w"), 1)
require.NoError(t, err)
assert.Equal(t, []string{"a/", "implicit/", "implicit/file", "x/", "x/w"}, all())
}
// TestListNoCleanup checks the deprecated --no-cleanup is
// --directory-markers.
func TestListNoCleanup(t *testing.T) {
opt := Opt
opt.NoCleanup = true
b := newListBackendOpt(t, opt, "bucket/a/b")
require.NoError(t, b.deleteObject(context.Background(), "bucket", "a/b"))
assert.Equal(t, []string{"a/"}, listPages(t, b, listPrefix("", ""), 1000))
}