"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
// PutObject error handling tests for serve s3.
//
// A PUT which fails part way through its body must never disturb the object
// already stored at the key.
package s3
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"path"
"sync"
"testing"
"time"
"github.com/minio/minio-go/v7"
"github.com/ncw/swift/v2"
"github.com/rclone/gofakes3"
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// errorReader yields data then fails with err.
type errorReader struct {
data []byte
err error
}
func (r *errorReader) Read(p []byte) (int, error) {
if len(r.data) == 0 {
return 0, r.err
}
n := copy(p, r.data)
r.data = r.data[n:]
return n, nil
}
// newPutTestBackend returns an s3Backend for direct PutObject tests, backed by
// the named remote, plus the backing Fs and a bucket created on it. vfsOpt
// overrides the VFS options (nil for the defaults).
func newPutTestBackend(t *testing.T, backing string, vfsOpt *vfscommon.Options) (*s3Backend, fs.Fs, string) {
fstest.Initialise()
ctx := context.Background()
if backing == "" {
backing = t.TempDir()
}
f, err := fs.NewFs(ctx, backing)
require.NoError(t, err)
bucket := fmt.Sprintf("test-%d", testBackingCounter.Add(1))
require.NoError(t, f.Mkdir(ctx, bucket))
if vfsOpt == nil {
vfsOpt = &vfscommon.Opt
}
// The VFS is cached per remote, so a shared ":memory:" backing reuses a
// VFS whose cached root listing predates the bucket just created; forget
// it so the new bucket is visible.
if root, err := vfs.New(ctx, f, vfsOpt).Root(); err == nil {
root.ForgetAll()
}
opt := Opt
opt.HTTP.ListenAddr = []string{endpoint}
w, err := newServer(ctx, f, &opt, vfsOpt, &proxy.Opt)
require.NoError(t, err)
t.Cleanup(func() { _ = w.Shutdown() })
return w.backend, f, bucket
}
var errBoom = errors.New("boom")
// failureModes enumerates the ways a PUT body can fail to arrive in full:
// an arbitrary read error, the unexpected EOF a dropped connection gives
// (which must not be mistaken for a clean end of stream), and a body which
// ends cleanly short of its declared size.
var failureModes = []struct {
name string
readerErr error
wantErr error
}{
{"ReadError", errBoom, errBoom},
{"UnexpectedEOF", io.ErrUnexpectedEOF, io.ErrUnexpectedEOF},
{"ShortBody", io.EOF, gofakes3.ErrIncompleteBody},
}
// failPut makes a PutObject call to bucket/object whose body fails with
// readerErr part way through and asserts wantErr is passed back.
func failPut(t *testing.T, b *s3Backend, bucket, object string, readerErr, wantErr error) {
_, err := b.PutObject(context.Background(), bucket, object, map[string]string{},
&errorReader{data: []byte(random.String(50)), err: readerErr}, 1000)
require.ErrorIs(t, err, wantErr)
}
// TestPutObjectFailurePreservesExisting checks that a failed PUT leaves the
// object stored at the key untouched - neither removed nor overwritten with
// truncated data - with no temporary object left behind.
func TestPutObjectFailurePreservesExisting(t *testing.T) {
for _, tc := range testRemotes {
for _, fm := range failureModes {
t.Run(tc.name+"/"+fm.name, func(t *testing.T) {
b, f, bucket := newPutTestBackend(t, tc.backing, nil)
ctx := context.Background()
const object = "existing.txt"
existing := []byte(random.String(100))
_, err := b.PutObject(ctx, bucket, object, map[string]string{}, bytes.NewReader(existing), int64(len(existing)))
require.NoError(t, err)
assert.Equal(t, existing, readObject(t, f, bucket, object))
requireOnly(t, f, bucket, object)
failPut(t, b, bucket, object, fm.readerErr, fm.wantErr)
assert.Equal(t, existing, readObject(t, f, bucket, object))
requireOnly(t, f, bucket, object)
// The object must still be served correctly - in particular
// HeadObject must not report the failed upload's size.
head, err := b.HeadObject(ctx, bucket, object)
require.NoError(t, err)
assert.Equal(t, int64(len(existing)), head.Size)
})
}
}
}
// TestPutObjectFailureNewKey checks that a failed PUT to a key with no
// existing object leaves nothing behind - no partial object at the key and
// no temporary object.
func TestPutObjectFailureNewKey(t *testing.T) {
for _, tc := range testRemotes {
for _, fm := range failureModes {
t.Run(tc.name+"/"+fm.name, func(t *testing.T) {
b, f, bucket := newPutTestBackend(t, tc.backing, nil)
ctx := context.Background()
const object = "new.txt"
failPut(t, b, bucket, object, fm.readerErr, fm.wantErr)
_, err := f.NewObject(ctx, path.Join(bucket, object))
require.ErrorIs(t, err, fs.ErrorObjectNotFound)
requireOnly(t, f, bucket)
// The failed upload must not be served either
_, err = b.HeadObject(ctx, bucket, object)
require.Error(t, err)
})
}
}
}
// TestPutObjectFailureNewDir checks that a failed PUT to a key in
// directories which don't exist removes the directories it made, which
// would otherwise be listed as directory markers.
func TestPutObjectFailureNewDir(t *testing.T) {
for _, tc := range testRemotes {
t.Run(tc.name, func(t *testing.T) {
b, _, bucket := newPutTestBackend(t, tc.backing, nil)
ctx := context.Background()
_, err := b.PutObject(ctx, bucket, "existing/file.txt", map[string]string{}, bytes.NewReader([]byte("hello")), 5)
require.NoError(t, err)
failPut(t, b, bucket, "existing/new/dirs/new.txt", errBoom, errBoom)
list, err := b.ListBucket(ctx, bucket, nil, gofakes3.ListBucketPage{})
require.NoError(t, err)
var keys []string
for _, content := range list.Contents {
keys = append(keys, content.Key)
}
assert.Equal(t, []string{"existing/file.txt"}, keys)
})
}
}
// TestPutObjectMtime checks that the object's modtime is set from the
// "X-Amz-Meta-Mtime" or "mtime" metadata supplied with the PUT, and is
// the Last-Modified of HEAD and GET.
func TestPutObjectMtime(t *testing.T) {
want := fstest.Time("2011-12-25T12:59:59.123456789Z")
for _, metaKey := range []string{"X-Amz-Meta-Mtime", "mtime"} {
t.Run(metaKey, func(t *testing.T) {
b, f, bucket := newPutTestBackend(t, "", nil)
ctx := context.Background()
const object = "mtime.txt"
contents := []byte(random.String(50))
// gofakes3 passes the time of the PUT as Last-Modified
meta := map[string]string{
metaKey: swift.TimeToFloatString(want),
"Last-Modified": formatHeaderTime(time.Now()),
}
_, err := b.PutObject(ctx, bucket, object, meta, bytes.NewReader(contents), int64(len(contents)))
require.NoError(t, err)
o, err := f.NewObject(ctx, path.Join(bucket, object))
require.NoError(t, err)
fstest.AssertTimeEqualWithPrecision(t, object, want, o.ModTime(ctx), f.Precision())
head, err := b.HeadObject(ctx, bucket, object)
require.NoError(t, err)
assert.Equal(t, formatHeaderTime(want), head.Metadata["Last-Modified"], "HEAD")
obj, err := b.GetObject(ctx, bucket, object, nil)
require.NoError(t, err)
require.NoError(t, obj.Contents.Close())
assert.Equal(t, formatHeaderTime(want), obj.Metadata["Last-Modified"], "GET")
})
}
}
// TestPutObjectLastModified checks that HeadObject and GetObject report
// the object's modification time as Last-Modified straight after a PUT.
// gofakes3 adds the time of the upload to the metadata it passes in, and
// that must not be cached in place of the object's own time.
func TestPutObjectLastModified(t *testing.T) {
want := fstest.Time("2021-06-06T06:06:06Z")
b, _, bucket := newPutTestBackend(t, "", nil)
ctx := context.Background()
const object = "last-modified.txt"
contents := []byte(random.String(50))
meta := map[string]string{
"X-Amz-Meta-Mtime": swift.TimeToFloatString(want),
"Last-Modified": formatHeaderTime(time.Now()),
}
_, err := b.PutObject(ctx, bucket, object, meta, bytes.NewReader(contents), int64(len(contents)))
require.NoError(t, err)
head, err := b.HeadObject(ctx, bucket, object)
require.NoError(t, err)
assert.Equal(t, formatHeaderTime(want), head.Metadata["Last-Modified"], "HeadObject")
get, err := b.GetObject(ctx, bucket, object, nil)
require.NoError(t, err)
_ = get.Contents.Close()
assert.Equal(t, formatHeaderTime(want), get.Metadata["Last-Modified"], "GetObject")
}
// waitForObject waits for bucket/object to appear on the backing Fs (e.g.
// after the VFS write-back delay).
func waitForObject(t *testing.T, f fs.Fs, bucket, object string) {
ctx := context.Background()
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if _, err := f.NewObject(ctx, path.Join(bucket, object)); err == nil {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatalf("object %s/%s never appeared on the backing remote", bucket, object)
}
// TestPutObjectFailureCached checks the failed PUT semantics with the cache
// modes that write through the VFS cache - minimal and above, since the PUT
// opens its handle read-write: the truncated file the failed upload leaves
// in the cache must be discarded, not written back over the object at the
// key.
func TestPutObjectFailureCached(t *testing.T) {
for _, cm := range []vfscommon.CacheMode{vfscommon.CacheModeMinimal, vfscommon.CacheModeWrites} {
for _, tc := range testRemotes {
t.Run(cm.String()+"/"+tc.name, func(t *testing.T) {
vfsOpt := vfscommon.Opt
vfsOpt.CacheMode = cm
vfsOpt.WriteBack = fs.Duration(100 * time.Millisecond)
b, f, bucket := newPutTestBackend(t, tc.backing, &vfsOpt)
const object = "cached.txt"
existing := []byte(random.String(100))
_, err := b.PutObject(context.Background(), bucket, object, map[string]string{}, bytes.NewReader(existing), int64(len(existing)))
require.NoError(t, err)
waitForObject(t, f, bucket, object)
assert.Equal(t, existing, readObject(t, f, bucket, object))
failPut(t, b, bucket, object, errBoom, errBoom)
// Wait out several write-back intervals to catch the truncated
// data being written back before checking the object is
// untouched.
time.Sleep(time.Second)
assert.Equal(t, existing, readObject(t, f, bucket, object))
requireOnly(t, f, bucket, object)
})
}
}
}
// TestDeleteObjectForgetsMetadata checks that deleting an object forgets
// its metadata, so it isn't kept in memory and doesn't reappear on a new
// object at the same key.
func TestDeleteObjectForgetsMetadata(t *testing.T) {
b, f, bucket := newPutTestBackend(t, "", nil)
ctx := context.Background()
const key = "meta.txt"
meta := map[string]string{"X-Amz-Meta-Colour": "blue"}
_, err := b.PutObject(ctx, bucket, key, meta, bytes.NewReader([]byte("one")), 3)
require.NoError(t, err)
obj, err := b.HeadObject(ctx, bucket, key)
require.NoError(t, err)
assert.Equal(t, "blue", obj.Metadata["X-Amz-Meta-Colour"])
_, err = b.DeleteObject(ctx, bucket, key)
require.NoError(t, err)
// Recreate the object without going through serve s3.
src := object.NewStaticObjectInfo(path.Join(bucket, key), time.Now(), 3, true, nil, nil)
_, err = f.Put(ctx, bytes.NewReader([]byte("two")), src)
require.NoError(t, err)
_vfs, err := b.s.getVFS(ctx)
require.NoError(t, err)
b.forgetPath(_vfs, path.Join(bucket, key))
obj, err = b.HeadObject(ctx, bucket, key)
require.NoError(t, err)
assert.NotContains(t, obj.Metadata, "X-Amz-Meta-Colour", "metadata of the deleted object reappeared")
}
// TestMetadataMaxObjects checks that the metadata of no more than
// --metadata-max-objects objects of a user is kept, forgetting that of
// the least recently used, and that one user's objects don't push out
// the metadata of another's.
func TestMetadataMaxObjects(t *testing.T) {
b, _, bucket := newPutTestBackend(t, "", nil)
b.s.opt.MetadataMaxObjects = 2
ctx, ctxOther := context.Background(), tenantCtx("other")
put := func(ctx context.Context, key string) {
meta := map[string]string{"X-Amz-Meta-Key": key}
_, err := b.PutObject(ctx, bucket, key, meta, bytes.NewReader([]byte(key)), int64(len(key)))
require.NoError(t, err)
}
head := func(ctx context.Context, key string) string {
obj, err := b.HeadObject(ctx, bucket, key)
require.NoError(t, err)
return obj.Metadata["X-Amz-Meta-Key"]
}
put(ctxOther, "other")
for _, key := range []string{"a", "b", "c"} {
put(ctx, key)
}
for key, want := range map[string]string{"a": "", "b": "b", "c": "c"} {
assert.Equal(t, want, head(ctx, key), key)
}
assert.Equal(t, "other", head(ctxOther, "other"), "another user's metadata was pushed out")
}
// TestMetadataAfterWriteback checks the metadata of an object written to
// the VFS cache is still returned once it has been written back to the
// backend.
func TestMetadataAfterWriteback(t *testing.T) {
b, _, bucket := newPutTestBackend(t, "", cacheWritesVFSOpt(100*time.Millisecond))
ctx := context.Background()
_vfs, err := b.s.getVFS(ctx)
require.NoError(t, err)
mtime := swift.TimeToFloatString(time.Now().Add(-time.Hour))
meta := map[string]string{"X-Amz-Meta-Colour": "red", "X-Amz-Meta-Mtime": mtime}
_, err = b.PutObject(ctx, bucket, "key", meta, bytes.NewReader([]byte("data")), 4)
require.NoError(t, err)
colour := func() string {
obj, err := b.HeadObject(ctx, bucket, "key")
require.NoError(t, err)
return obj.Metadata["X-Amz-Meta-Colour"]
}
assert.Equal(t, "red", colour(), "before writeback")
_vfs.WaitForWriters(10 * time.Second)
b.forgetPath(_vfs, path.Join(bucket, "key"))
assert.Equal(t, "red", colour(), "after writeback")
}
// TestConditionalPutRace checks that of many concurrent PUTs of a new
// object with If-None-Match: *, which all find the object missing when
// they start, only one writes it.
func TestConditionalPutRace(t *testing.T) {
core, f, bucket := newMultipartTestServer(t, false)
ctx := context.Background()
const object = "race.bin"
const n = 20
data := []byte(random.String(256 * 1024))
var wg sync.WaitGroup
var start sync.WaitGroup
start.Add(1)
errs := make([]error, n)
for i := range n {
wg.Add(1)
go func() {
defer wg.Done()
opts := minio.PutObjectOptions{}
opts.SetMatchETagExcept("*")
start.Wait()
_, errs[i] = core.PutObject(ctx, bucket, object, bytes.NewReader(data), int64(len(data)), "", "", opts)
}()
}
start.Done()
wg.Wait()
succeeded := 0
for _, err := range errs {
if err == nil {
succeeded++
} else {
assert.Equal(t, "PreconditionFailed", minio.ToErrorResponse(err).Code, "%v", err)
}
}
assert.Equal(t, 1, succeeded)
assert.Equal(t, data, readObject(t, f, bucket, object))
requireOnly(t, f, bucket, object)
}