"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
mega: wait for server events after upload, delete and move
Nick Craig-Woodd489c59a59d2
diff --git a/backend/mega/mega.go b/backend/mega/mega.goindex b0ef2e10b..297f62436 100644--- a/backend/mega/mega.go+++ b/backend/mega/mega.go@@ -770,6 +770,8 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote streturn fmt.Errorf("server-side move failed to lookup src parent dir: %w", err)}+ waitEvent := f.srv.WaitEventsStart()+// move the object into its new directory if requiredif srcDirNode != dstDirNode && srcDirNode.GetHash() != dstDirNode.GetHash() {//log.Printf("move src %p %q dst %p %q", srcDirNode, srcDirNode.GetName(), dstDirNode, dstDirNode.GetName())@@ -782,8 +784,6 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st}}- waitEvent := f.srv.WaitEventsStart()-// rename the object if requiredif srcLeaf != dstLeaf {//log.Printf("rename %q to %q", srcLeaf, dstLeaf)@@ -1216,6 +1216,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op}}+ waitEvent := o.fs.srv.WaitEventsStart()+// Finish the uploadvar info *mega.Nodeerr = o.fs.pacer.Call(func() (bool, error) {@@ -1235,15 +1237,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opo.info = nil}+ // Wait for the server to confirm the new file+ o.fs.srv.WaitEvents(waitEvent, eventWaitTime)+return o.setMetaData(info)}// Remove an objectfunc (o *Object) Remove(ctx context.Context) error {+ waitEvent := o.fs.srv.WaitEventsStart()+err := o.fs.deleteNode(ctx, o.info)if err != nil {return fmt.Errorf("Remove object failed: %w", err)}++ // Wait for the server to confirm the deletion+ o.fs.srv.WaitEvents(waitEvent, eventWaitTime)return nil}diff --git a/backend/mega/mega_internal_test.go b/backend/mega/mega_internal_test.gonew file mode 100644index 000000000..c39dfff4d--- /dev/null+++ b/backend/mega/mega_internal_test.go@@ -0,0 +1,63 @@+package mega++import (+ "bytes"+ "context"+ "testing"+ "time"++ "github.com/rclone/rclone/fs/object"+ "github.com/rclone/rclone/fstest/fstests"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// InternalTestGhostAfterRemove checks that a file removed in a long-running+// process disappears from listings straight away rather than lingering as a+// ghost until go-mega's next event poll reconciles the in-memory tree.+//+// The ghost is a race between the upload's server event being replayed and+// the delete, so it puts, removes and lists in a tight loop to make it show+// up reliably.+func (f *Fs) InternalTestGhostAfterRemove(t *testing.T) {+ ctx := context.Background()++ // Exercise the default soft-delete path - the hard_delete=true path+ // hits a separate bug in go-mega's Delete which leaves the node in its+ // parent's children regardless of what rclone does.+ if f.opt.HardDelete {+ oldHardDelete := f.opt.HardDelete+ f.opt.HardDelete = false+ defer func() { f.opt.HardDelete = oldHardDelete }()+ }++ dir := "ghost-test"+ require.NoError(t, f.Mkdir(ctx, dir))+ defer func() { _ = f.Rmdir(ctx, dir) }()++ contents := []byte("ghost test contents")+ for i := range 10 {+ remote := dir + "/test.txt"+ src := object.NewStaticObjectInfo(remote, time.Now(), int64(len(contents)), true, nil, nil)+ obj, err := f.Put(ctx, bytes.NewReader(contents), src)+ require.NoError(t, err)+ require.NoError(t, obj.Remove(ctx))++ // The file must be gone from the listing straight away - if the+ // in-memory tree isn't settled it lingers as a ghost.+ entries, err := f.List(ctx, dir)+ require.NoError(t, err)+ names := make([]string, 0, len(entries))+ for _, entry := range entries {+ names = append(names, entry.Remote())+ }+ assert.NotContains(t, names, remote, "file should be gone immediately after remove (iteration %d)", i)+ }+}++// InternalTest dispatches the backend specific internal tests+func (f *Fs) InternalTest(t *testing.T) {+ t.Run("GhostAfterRemove", f.InternalTestGhostAfterRemove)+}++var _ fstests.InternalTester = (*Fs)(nil)