"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fstests: test uploads survive a transient HTTP error and return pooled buffers
Nick Craig-Wood05ac69e123b9
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.goindex 4a773b3d3..a0fd66254 100644--- a/fstest/fstests/fstests.go+++ b/fstest/fstests/fstests.go@@ -13,6 +13,8 @@ import ("fmt""io""math/bits"+ "net"+ "net/http""os""path""path/filepath"@@ -21,6 +23,7 @@ import ("sort""strconv""strings"+ "sync/atomic""testing""time"@@ -28,6 +31,7 @@ import ("github.com/rclone/rclone/fs/cache""github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/fspath""github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/object"@@ -36,6 +40,7 @@ import ("github.com/rclone/rclone/fstest""github.com/rclone/rclone/fstest/testserver""github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/pool""github.com/rclone/rclone/lib/random""github.com/rclone/rclone/lib/readers""github.com/stretchr/testify/assert"@@ -250,6 +255,7 @@ func testPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item,obj fs.ObjectuploadHash *hash.MultiHasher)+ inUse := pool.Global().InUse()retry(t, "PutLarge", func() error {r := readers.NewPatternReader(file.Size)uploadHash = hash.NewMultiHasher()@@ -266,6 +272,7 @@ func testPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item,}return err})+ assert.Equal(t, inUse, pool.Global().InUse(), "buffers not returned to the global pool after upload")file.Hashes = uploadHash.Sums()file.Check(t, obj, f.Precision())@@ -2544,6 +2551,187 @@ func Run(t *testing.T, opt *Opt) {}})+ // TestFsPutRetry tests that an upload survives a transient HTTP+ // failure without corrupting the object+ t.Run("FsPutRetry", func(t *testing.T) {+ skipIfNotOk(t)++ // faults are the transient failures injected into the+ // first request carrying upload data+ faults := []struct {+ name string+ statusCode int+ err error+ }{+ {name: "Status500", statusCode: http.StatusInternalServerError},+ // io.ErrUnexpectedEOF is in fserrors' portable list of+ // retriable errors, unlike syscall.ECONNRESET which+ // doesn't exist on plan9.+ {name: "NetError", err: &net.OpError{Op: "write", Net: "tcp", Err: io.ErrUnexpectedEOF}},+ }++ // putRetry uploads size bytes to remote, failing the first+ // request carrying at least minBody bytes of body with the+ // given fault before it is sent. Put must then either+ // succeed with the correct contents or fail cleanly, and+ // any pooled buffers it used must be returned.+ putRetry := func(t *testing.T, remote string, size, minBody int64, statusCode int, faultErr error) {+ var faulted atomic.Bool+ fshttp.SetFaultInjector(func(req *http.Request) (int, error) {+ if faulted.Load() {+ return 0, nil+ }+ if req.Body == nil || req.Body == http.NoBody {+ return 0, nil+ }+ if req.ContentLength > 0 {+ if req.ContentLength < minBody {+ return 0, nil+ }+ } else {+ // The length is unknown (e.g. a multipart+ // form) so peek at the body to see whether it+ // carries at least minBody bytes of data,+ // replaying the peeked bytes if the request is+ // sent on. This stops the fault being spent on+ // a small API call sent with unknown length.+ peek := make([]byte, minBody)+ n, _ := io.ReadFull(req.Body, peek)+ req.Body = struct {+ io.Reader+ io.Closer+ }{io.MultiReader(bytes.NewReader(peek[:n]), req.Body), req.Body}+ if int64(n) < minBody {+ return 0, nil+ }+ }+ if !faulted.CompareAndSwap(false, true) {+ return 0, nil+ }+ return statusCode, faultErr+ })+ defer fshttp.SetFaultInjector(nil)++ file := fstest.Item{+ ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),+ Path: remote,+ Size: size,+ }+ // Remove anything left behind if the test fails part way+ t.Cleanup(func() {+ obj, err := f.NewObject(ctx, remote)+ if err == nil {+ _ = obj.Remove(ctx)+ }+ })+ inUse := pool.Global().InUse()+ uploadHash := hash.NewMultiHasher()+ in := io.TeeReader(readers.NewPatternReader(size), uploadHash)+ obji := object.NewStaticObjectInfo(remote, file.ModTime, size, true, nil, nil)+ obj, err := f.Put(ctx, in, obji)+ fshttp.SetFaultInjector(nil)+ assert.Equal(t, inUse, pool.Global().InUse(), "buffers not returned to the global pool after upload")+ file.Hashes = uploadHash.Sums()++ // checkAndRemove checks obj has the expected contents+ // then removes it+ checkAndRemove := func(obj fs.Object) {+ file.Check(t, obj, f.Precision())+ downloadHash := hash.NewMultiHasher()+ download, err := obj.Open(ctx)+ require.NoError(t, err)+ n, err := io.Copy(downloadHash, download)+ require.NoError(t, err)+ assert.Equal(t, size, n)+ require.NoError(t, download.Close())+ assert.Equal(t, file.Hashes, downloadHash.Sums())+ assert.NoError(t, obj.Remove(ctx))+ }++ if !faulted.Load() {+ if err == nil {+ assert.NoError(t, obj.Remove(ctx))+ }+ t.Skip("no upload request could be faulted")+ }+ if err != nil {+ // A clean failure is acceptable but the object must+ // not have been left behind with the wrong contents+ t.Logf("Put returned an error after the injected failure (acceptable): %v", err)+ obj, err := f.NewObject(ctx, remote)+ if errors.Is(err, fs.ErrorObjectNotFound) {+ return+ }+ require.NoError(t, err)+ checkAndRemove(obj)+ return+ }+ require.NotNil(t, obj)+ file.Check(t, obj, f.Precision())+ // Re-read the object, which waits for it to become visible+ checkAndRemove(fstest.NewObject(ctx, t, f, remote))+ }++ t.Run("Simple", func(t *testing.T) {+ var size int64 = 64 * 1024+ if *fstest.SizeLimit > 0 && size > *fstest.SizeLimit {+ size = *fstest.SizeLimit+ t.Logf("Reduce file size due to limit %d", size)+ }+ for _, fault := range faults {+ t.Run(fault.name, func(t *testing.T) {+ putRetry(t, "retry-"+fault.name+".bin", size, size, fault.statusCode, fault.err)+ })+ }+ })++ t.Run("Chunked", func(t *testing.T) {+ if testing.Short() {+ t.Skip("not running with -short")+ }+ if opt.ChunkedUpload.Skip {+ t.Skip("skipping as ChunkedUpload.Skip is set")+ }+ setUploadChunkSizer, _ := f.(SetUploadChunkSizer)+ if setUploadChunkSizer == nil {+ t.Skipf("%T does not implement SetUploadChunkSizer", f)+ }+ setUploadCutoffer, _ := f.(SetUploadCutoffer)++ // Use a chunk size large enough that only chunk+ // uploads, not API calls, carry enough body to fault+ chunkSize := max(opt.ChunkedUpload.MinChunkSize, 64*fs.Kibi)+ if opt.ChunkedUpload.MaxChunkSize > 0 && chunkSize > opt.ChunkedUpload.MaxChunkSize {+ chunkSize = opt.ChunkedUpload.MaxChunkSize+ }+ if opt.ChunkedUpload.CeilChunkSize != nil {+ chunkSize = opt.ChunkedUpload.CeilChunkSize(chunkSize)+ }++ oldChunkSize, err := setUploadChunkSizer.SetUploadChunkSize(chunkSize)+ require.NoError(t, err)+ var oldUploadCutoff fs.SizeSuffix+ if setUploadCutoffer != nil {+ oldUploadCutoff, err = setUploadCutoffer.SetUploadCutoff(chunkSize)+ require.NoError(t, err)+ }+ defer func() {+ _, err := setUploadChunkSizer.SetUploadChunkSize(oldChunkSize)+ assert.NoError(t, err)+ if setUploadCutoffer != nil {+ _, err := setUploadCutoffer.SetUploadCutoff(oldUploadCutoff)+ assert.NoError(t, err)+ }+ }()++ for _, fault := range faults {+ t.Run(fault.name, func(t *testing.T) {+ putRetry(t, "retry-chunked-"+fault.name+".bin", int64(2*chunkSize+1), int64(chunkSize/2), fault.statusCode, fault.err)+ })+ }+ })+ })+// TestFsPutShortEOF tests uploading a file where the reader// returns EOF before the declared size worth of data has been// read.