"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: add WriteFileHandle.CloseWithError to abandon streaming writes
Nick Craig-Wood2f0657c35b0f
diff --git a/vfs/write.go b/vfs/write.goindex cc1578494..b370d148a 100644--- a/vfs/write.go+++ b/vfs/write.go@@ -1,6 +1,9 @@package vfsimport (+ "context"+ "errors"+ "fmt""io""os""sync"@@ -21,6 +24,8 @@ type WriteFileHandle struct {o fs.Objectresult chan errorfile *File+ ctx context.Context // the streaming upload runs under this+ cancel context.CancelFunc // fails the streaming upload, unblocking a stalled writeoffset int64flags intclosed bool // set if handle has been closed@@ -46,6 +51,7 @@ func newWriteFileHandle(d *Dir, f *File, remote string, flags int) (*WriteFileHaresult: make(chan error, 1),file: f,}+ fh.ctx, fh.cancel = context.WithCancel(f.ctx)fh.cond = sync.Cond{L: &fh.mu}fh.file.addWriter(fh)return fh, nil@@ -85,7 +91,7 @@ func (fh *WriteFileHandle) openPending() (err error) {}()defer vfscommon.RecoverPanic(fh.remote, &err)// NB Rcat deals with Stats.Transferring, etc.- o, err = operations.Rcat(fh.file.ctx, fh.file.Fs(), fh.remote, pipeReader, time.Now(), nil)+ o, err = operations.Rcat(fh.ctx, fh.file.Fs(), fh.remote, pipeReader, time.Now(), nil)if err != nil {fs.Errorf(fh.remote, "WriteFileHandle.New Rcat failed: %v", err)}@@ -196,10 +202,22 @@ func (fh *WriteFileHandle) Offset() (offset int64) {//// Must be called with fh.mu heldfunc (fh *WriteFileHandle) close() (err error) {+ return fh.closeWithError(nil)+}++// closeWithError closes the file handle like close does. If reason is+// non-nil the streaming upload is failed with reason rather than being+// ended normally, so a partially written file is discarded instead of+// being stored as if it were complete.+//+// Must be called with fh.mu held+func (fh *WriteFileHandle) closeWithError(reason error) (err error) {if fh.closed {return ECLOSED}fh.closed = true+ // Release the upload context once the upload has finished+ defer fh.cancel()// leave writer open until file is transferreddefer func() {fh.file.delWriter(fh)@@ -211,14 +229,21 @@ func (fh *WriteFileHandle) close() (err error) {if err = fh.openPending(); err != nil {return err}- writeCloseErr := fh.pipeWriter.Close()+ writeCloseErr := fh.pipeWriter.CloseWithError(reason)err = <-fh.resultif err == nil {fh.file.setObject(fh.o)err = writeCloseErr- } else if fh.file.getObject() == nil {- // Remove vfs file entry when no object is present- _ = fh.file.Remove()+ } else {+ if fh.file.getObject() == nil {+ // Remove vfs file entry when no object is present+ _ = fh.file.Remove()+ }+ if reason != nil {+ // The upload's own error is usually just the cancellation+ // the abandon caused - reason is the cause worth reporting.+ err = reason+ }}return err}@@ -230,6 +255,34 @@ func (fh *WriteFileHandle) Close() error {return fh.close()}+// CloseWithError closes the file handle, abandoning the write.+//+// The streaming upload to the backend is failed with reason rather than+// being ended normally, so the partially written file is discarded+// instead of being stored as if it were complete. A stalled upload is+// interrupted rather than waited for, so this returns promptly even+// when the backend has stopped accepting data. A nil reason is the+// same as Close.+//+// It returns ECLOSED if the handle has already been closed.+func (fh *WriteFileHandle) CloseWithError(reason error) error {+ // Don't let an EOF-like reason be mistaken for a clean end of stream+ // by whatever is reading the other end of the pipe.+ if reason != nil && (errors.Is(reason, io.EOF) || errors.Is(reason, io.ErrUnexpectedEOF)) {+ reason = fmt.Errorf("write aborted: %v", reason)+ }+ if reason != nil {+ // Fail the upload before taking the lock: a write blocked on a+ // stalled upload holds fh.mu, and only failing the upload+ // unblocks it - otherwise the abandon would wait out the+ // backend's timeout (or forever without one).+ fh.cancel()+ }+ fh.mu.Lock()+ defer fh.mu.Unlock()+ return fh.closeWithError(reason)+}+// Flush is called on each close() of a file descriptor. So if a// filesystem wants to return write errors in close() and the file has// cached dirty data, this is a good place to write back data anddiff --git a/vfs/write_test.go b/vfs/write_test.goindex 77173bdf5..518bc2b2c 100644--- a/vfs/write_test.go+++ b/vfs/write_test.go@@ -13,6 +13,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fstest""github.com/rclone/rclone/lib/random"+ "github.com/rclone/rclone/vfs/vfscommon""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -235,6 +236,47 @@ func TestWriteFileHandleWriteAt(t *testing.T) {fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{}, fs.ModTimeNotSupported)}+func TestWriteFileHandleCloseWithError(t *testing.T) {+ // io.ErrUnexpectedEOF must fail the upload like any other reason.+ for _, reason := range []error{errors.New("upload interrupted"), io.ErrUnexpectedEOF} {+ t.Run(reason.Error(), func(t *testing.T) {+ r, vfs, fh := writeHandleCreate(t)++ // Write some data then abandon the write mid-stream+ n, err := fh.Write([]byte("hello"))+ require.NoError(t, err)+ assert.Equal(t, 5, n)++ err = fh.CloseWithError(reason)+ require.Error(t, err)+ assert.ErrorContains(t, err, reason.Error())++ // Check double close+ assert.Equal(t, ECLOSED, fh.Close())++ if *fstest.RemoteName == "" {+ _, err = vfs.Stat("file1")+ assert.Equal(t, ENOENT, err, "The abandoned file must not exist in the VFS or on the remote")+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.ModTimeNotSupported)+ }+ })+ }++ // CloseWithError with a nil reason behaves like Close and commits the file+ t.Run("NilReason", func(t *testing.T) {+ r, vfs := newTestVFS(t)+ h, err := vfs.OpenFile("file2", os.O_WRONLY|os.O_CREATE, 0777)+ require.NoError(t, err)+ fh2, ok := h.(*WriteFileHandle)+ require.True(t, ok)+ _, err = fh2.Write([]byte("hello"))+ require.NoError(t, err)+ require.NoError(t, fh2.CloseWithError(nil))+ file2 := fstest.NewItem("file2", "hello", t1)+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file2}, []string{}, fs.ModTimeNotSupported)+ })+}+func TestWriteFileHandleFlush(t *testing.T) {_, vfs, fh := writeHandleCreate(t)@@ -383,3 +425,83 @@ func TestFileReadAtZeroLength(t *testing.T) {func TestFileReadAtNonZeroLength(t *testing.T) {testFileReadAt(t, 100)}++// stallingFs wraps an Fs so that streaming uploads accept a little data and+// then stall until their context is cancelled, like a remote that has+// stopped accepting data.+type stallingFs struct {+ fs.Fs+ stalled chan struct{} // closed once the upload has stalled+ features *fs.Features+}++func newStallingFs(base fs.Fs) *stallingFs {+ f := &stallingFs{Fs: base, stalled: make(chan struct{})}+ features := *base.Features()+ features.PutStream = f.PutStream+ f.features = &features+ return f+}++// Features returns the optional features of this Fs+func (f *stallingFs) Features() *fs.Features { return f.features }++// PutStream reads a little of in then stalls until ctx is cancelled.+func (f *stallingFs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ buf := make([]byte, 1024)+ _, _ = in.Read(buf)+ close(f.stalled)+ <-ctx.Done()+ return nil, ctx.Err()+}++// TestWriteFileHandleCloseWithErrorStalled checks that CloseWithError+// interrupts an upload the backend has stopped accepting data for. The+// writer is blocked in the pipe holding the handle's lock, so the abandon+// must fail the upload to unblock it, rather than waiting on the lock+// until the backend gives up of its own accord.+func TestWriteFileHandleCloseWithErrorStalled(t *testing.T) {+ r := fstest.NewRun(t)+ f := newStallingFs(r.Fremote)+ // A tweaked option so this VFS is not deduplicated onto an active VFS+ // of the unwrapped remote, whose config string it shares.+ opt := vfscommon.Opt+ opt.WriteWait += fs.Duration(time.Millisecond)+ vfs := New(context.Background(), f, &opt)+ t.Cleanup(vfs.Shutdown)++ h, err := vfs.OpenFile("stalled", os.O_WRONLY|os.O_CREATE, 0777)+ require.NoError(t, err)+ fh, ok := h.(*WriteFileHandle)+ require.True(t, ok)++ // Write more than the transfer buffers absorb, so the write blocks on+ // the stalled upload while holding the handle's lock.+ writeDone := make(chan error, 1)+ go func() {+ _, err := fh.Write(make([]byte, 20<<20))+ writeDone <- err+ }()++ select {+ case <-f.stalled:+ case <-time.After(10 * time.Second):+ t.Fatal("upload never started")+ }++ // The abandon must interrupt the stalled upload promptly.+ closeDone := make(chan error, 1)+ go func() { closeDone <- fh.CloseWithError(errors.New("abandoned")) }()+ select {+ case err := <-closeDone:+ require.Error(t, err)+ case <-time.After(10 * time.Second):+ t.Fatal("CloseWithError did not interrupt the stalled upload")+ }+ select {+ case err := <-writeDone:+ require.Error(t, err)+ case <-time.After(10 * time.Second):+ t.Fatal("the blocked write was never unblocked")+ }+}