"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
memory: fix deadlock in operations.Purge
nielash2b0a25a64d95
diff --git a/backend/memory/memory.go b/backend/memory/memory.goindex 11c02d78a..608f42292 100644--- a/backend/memory/memory.go+++ b/backend/memory/memory.go@@ -38,8 +38,7 @@ func init() {}// Options defines the configuration for this backend-type Options struct {-}+type Options struct{}// Fs represents a remote memory servertype Fs struct {@@ -385,10 +384,22 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err efunc (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {bucket, directory := f.split(dir)list := walk.NewListRHelper(callback)+ entries := fs.DirEntries{}listR := func(bucket, directory, prefix string, addBucket bool) error {- return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error {- return list.Add(entry)+ err = f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error {+ entries = append(entries, entry) // can't list.Add here -- could deadlock+ return nil})+ if err != nil {+ return err+ }+ for _, entry := range entries {+ err = list.Add(entry)+ if err != nil {+ return err+ }+ }+ return nil}if bucket == "" {entries, err := f.listBuckets(ctx)diff --git a/backend/memory/memory_internal_test.go b/backend/memory/memory_internal_test.gonew file mode 100644index 000000000..fe8db1f57--- /dev/null+++ b/backend/memory/memory_internal_test.go@@ -0,0 +1,40 @@+package memory++import (+ "context"+ "fmt"+ "testing"++ _ "github.com/rclone/rclone/backend/local"+ "github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/fstest"+ "github.com/rclone/rclone/fstest/fstests"+ "github.com/stretchr/testify/require"+)++var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")++// InternalTest dispatches all internal tests+func (f *Fs) InternalTest(t *testing.T) {+ t.Run("PurgeListDeadlock", func(t *testing.T) {+ testPurgeListDeadlock(t)+ })+}++// test that Purge fallback does not result in deadlock from concurrently listing and removing+func testPurgeListDeadlock(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRunIndividual(t)+ r.Mkdir(ctx, r.Fremote)+ r.Fremote.Features().Disable("Purge") // force fallback-purge++ // make a lot of files to prevent it from finishing too quickly+ for i := 0; i < 100; i++ {+ dst := "file" + fmt.Sprint(i) + ".txt"+ r.WriteObject(ctx, dst, "hello", t1)+ }++ require.NoError(t, operations.Purge(ctx, r.Fremote, ""))+}++var _ fstests.InternalTester = (*Fs)(nil)