"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: don't crash the process if a backend panics on a background goroutine GHSA-6jcg-q3wp-x2f4
Nick Craig-Wood208d7df87758
diff --git a/vfs/vfs.go b/vfs/vfs.goindex aaf94d9bf..e1a118982 100644--- a/vfs/vfs.go+++ b/vfs/vfs.go@@ -287,6 +287,7 @@ func New(ctx context.Context, f fs.Fs, opt *vfscommon.Options) *VFS {// refresh the directory cache for all directoriesfunc (vfs *VFS) refresh() {+ defer vfscommon.RecoverPanic(vfs.f, nil)fs.Debugf(vfs.f, "Refreshing VFS directory cache")err := vfs.root.readDirTree()if err != nil {diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.goindex 66852e9ec..eaa17b8a0 100644--- a/vfs/vfscache/cache.go+++ b/vfs/vfscache/cache.go@@ -789,6 +789,9 @@ func (c *Cache) purgeOverQuota() {// clean empties the cache of stuff if it canfunc (c *Cache) clean(kicked bool) {+ // Recover here rather than in the cleaner goroutine so that a panic+ // cleaning one item does not stop the cache being cleaned ever again.+ defer vfscommon.RecoverPanic(c.fremote, nil)// Cache may be empty so end_, err := os.Stat(c.root)if os.IsNotExist(err) {diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.goindex 495fdf2bf..333edc8fc 100644--- a/vfs/vfscache/downloaders/downloaders.go+++ b/vfs/vfscache/downloaders/downloaders.go@@ -406,6 +406,7 @@ func (dls *Downloaders) _dispatchWaiters() {// Send any waiters which have completed back to their callers and make sure// there is a downloader appropriate for each waiterfunc (dls *Downloaders) kickWaiters() (err error) {+ defer vfscommon.RecoverPanic(dls.src, &err)dls.mu.Lock()defer dls.mu.Unlock()@@ -556,6 +557,7 @@ func (dl *downloader) open(offset int64) (err error) {// close the downloaderfunc (dl *downloader) close(inErr error) (err error) {// defer log.Trace(dl.dls.src, "inErr=%v", err)("err=%v", &err)+ defer vfscommon.RecoverPanic(dl.dls.src, &err)checkErr := func(e error) {if e == nil || errors.Is(err, asyncreader.ErrorStreamAbandoned) {return@@ -563,17 +565,20 @@ func (dl *downloader) close(inErr error) (err error) {err = e}dl.mu.Lock()- if dl.in != nil {- checkErr(dl.in.Close())- dl.in = nil+ defer dl.mu.Unlock()+ // Mark closed and detach the reader and transfer before closing them:+ // closing the reader runs backend code, and if that panics the recover+ // above must not leave the downloader locked or looking still open.+ dl._closed = true+ in, tr := dl.in, dl.tr+ dl.in, dl.tr = nil, nil+ if in != nil {+ checkErr(in.Close())}- if dl.tr != nil {- dl.tr.Done(dl.dls.ctx, inErr)- dl.tr = nil+ if tr != nil {+ tr.Done(dl.dls.ctx, inErr)}- dl._closed = true- dl.mu.Unlock()- return nil+ return err}// closed returns true if the downloader has been closed already@@ -622,6 +627,7 @@ func (dl *downloader) stopAndClose(inErr error) (err error) {// Start downloading to the local file starting at offset until maxOffset.func (dl *downloader) download() (n int64, err error) {+ defer vfscommon.RecoverPanic(dl.dls.src, &err)// defer log.Trace(dl.dls.src, "")("err=%v", &err)n, err = dl.in.WriteTo(dl)if err != nil && !errors.Is(err, asyncreader.ErrorStreamAbandoned) {diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.goindex 7e643c054..475f97513 100644--- a/vfs/vfscache/item.go+++ b/vfs/vfscache/item.go@@ -17,6 +17,7 @@ import ("github.com/rclone/rclone/lib/ranges""github.com/rclone/rclone/vfs/vfscache/downloaders""github.com/rclone/rclone/vfs/vfscache/writeback"+ "github.com/rclone/rclone/vfs/vfscommon")// NB as Cache and Item are tightly linked it is necessary to have a@@ -721,6 +722,7 @@ func (item *Item) Close(storeFn StoreFn) (err error) {// Grace period only applies to non-dirty files, so storeFn (only// needed for writeback) and syncWriteBack are not relevant.func (item *Item) closeAfterGrace() {+ defer vfscommon.RecoverPanic(item.name, nil)item.mu.Lock()defer item.mu.Unlock()@@ -736,9 +738,14 @@ func (item *Item) closeAfterGrace() {// progress so a concurrent open waits rather than tripping over the// half-closed handle.item.closing = make(chan struct{})+ // Release the waiters however this ends: if a panic in the backend is+ // recovered above, leaving item.closing open would block every later+ // open of this file forever.+ defer func() {+ close(item.closing)+ item.closing = nil+ }()err := item._actualClose(nil, false)- close(item.closing)- item.closing = nilif err != nil {fs.Errorf(item.name, "vfs cache: close after grace period failed: %v", err)}diff --git a/vfs/vfscache/writeback/writeback.go b/vfs/vfscache/writeback/writeback.goindex 76acbb446..3e3575158 100644--- a/vfs/vfscache/writeback/writeback.go+++ b/vfs/vfscache/writeback/writeback.go@@ -352,7 +352,9 @@ func (wb *WriteBack) upload(ctx context.Context, wbItem *writeBackItem) {fs.Debugf(wbItem.name, "vfs cache: starting upload")wb.mu.Unlock()- err := putFn(ctx)+ // Recover around the upload itself rather than the whole goroutine, so a+ // panicking backend is retried like any other upload failure below.+ err := vfscommon.RecoverCall(wbItem.name, func() error { return putFn(ctx) })wb.mu.Lock()wbItem.cancel() // cancel context to release resources since store donediff --git a/vfs/vfscommon/recover.go b/vfs/vfscommon/recover.gonew file mode 100644index 000000000..1b630c208--- /dev/null+++ b/vfs/vfscommon/recover.go@@ -0,0 +1,29 @@+package vfscommon++import (+ "fmt"+ "runtime/debug"++ "github.com/rclone/rclone/fs"+)++// RecoverPanic recovers a panic, logs it against o with a stack trace and,+// if err is not nil, stores it there. Use it as `defer RecoverPanic(o, &err)`.+func RecoverPanic(o any, err *error) {+ if r := recover(); r != nil {+ fs.Errorf(o, "panic: %v\n%s", r, debug.Stack())+ if err != nil {+ *err = fmt.Errorf("panic: %v", r)+ }+ }+}++// RecoverCall calls fn, converting a panic into an error.+//+// Prefer this to recovering a whole goroutine where the caller already+// handles an error, so a panic is retried or reported the same way a failure+// is, rather than abandoning the surrounding work part way through.+func RecoverCall(o any, fn func() error) (err error) {+ defer RecoverPanic(o, &err)+ return fn()+}diff --git a/vfs/vfscommon/recover_test.go b/vfs/vfscommon/recover_test.gonew file mode 100644index 000000000..01bd7c421--- /dev/null+++ b/vfs/vfscommon/recover_test.go@@ -0,0 +1,41 @@+package vfscommon++import (+ "errors"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestRecoverPanic(t *testing.T) {+ // Panic recovered into err+ err := func() (err error) {+ defer RecoverPanic("test", &err)+ panic("boom")+ }()+ require.Error(t, err)+ assert.ErrorContains(t, err, "boom")++ // No panic leaves err alone+ err = func() (err error) {+ defer RecoverPanic("test", nil)+ return nil+ }()+ assert.NoError(t, err)++ // A nil error target is allowed, for goroutines with nothing to report to+ assert.NotPanics(t, func() {+ defer RecoverPanic("test", nil)+ panic("boom")+ })+}++func TestRecoverCall(t *testing.T) {+ assert.ErrorContains(t, RecoverCall("test", func() error { panic("boom") }), "boom")++ // Ordinary errors and results pass through untouched+ errFoo := errors.New("foo")+ assert.Equal(t, errFoo, RecoverCall("test", func() error { return errFoo }))+ assert.NoError(t, RecoverCall("test", func() error { return nil }))+}diff --git a/vfs/write.go b/vfs/write.goindex fcc700edb..cc1578494 100644--- a/vfs/write.go+++ b/vfs/write.go@@ -8,6 +8,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/vfs/vfscommon")// WriteFileHandle is an open for write handle on a File@@ -69,15 +70,25 @@ func (fh *WriteFileHandle) openPending() (err error) {var pipeReader *io.PipeReaderpipeReader, fh.pipeWriter = io.Pipe()go func() {+ var (+ o fs.Object+ err error+ )+ // Report the outcome however this ends: if a panic escaped here the+ // process would die, and if it were recovered without reporting then+ // whoever is waiting on the result would wait forever.+ defer func() {+ // Close the pipeReader so the pipeWriter fails with ErrClosedPipe+ _ = pipeReader.Close()+ fh.o = o+ fh.result <- err+ }()+ defer vfscommon.RecoverPanic(fh.remote, &err)// NB Rcat deals with Stats.Transferring, etc.- o, err := operations.Rcat(fh.file.ctx, fh.file.Fs(), fh.remote, pipeReader, time.Now(), nil)+ o, err = operations.Rcat(fh.file.ctx, fh.file.Fs(), fh.remote, pipeReader, time.Now(), nil)if err != nil {fs.Errorf(fh.remote, "WriteFileHandle.New Rcat failed: %v", err)}- // Close the pipeReader so the pipeWriter fails with ErrClosedPipe- _ = pipeReader.Close()- fh.o = o- fh.result <- err}()fh.file.setSize(0)fh.truncated = true