Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: fix hang and lost context when moving a directory file by file

Nick Craig-Woodb82a024de00d
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index cc878c2e5..8aaf81ca4 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1594,7 +1594,7 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
}
fs.Debugf(nil, "removing %d level %d directories", len(dirs), level)
sort.Strings(dirs)
- g, gCtx := errgroup.WithContext(context.Background())
+ g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(ci.Checkers)
for _, dir := range dirs {
// End early if error
@@ -2514,11 +2514,17 @@ func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err err
return nil
})
}
+sending:
for dir, entries := range tree {
dstPath := dstRemote + dir[len(srcRemote):]
for _, entry := range entries {
if o, ok := entry.(fs.Object); ok {
- renames <- rename{o, path.Join(dstPath, path.Base(o.Remote()))}
+ select {
+ case renames <- rename{o, path.Join(dstPath, path.Base(o.Remote()))}:
+ case <-gCtx.Done():
+ // The workers have stopped so stop sending
+ break sending
+ }
}
}
}
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 6e0045a62..debacc0ab 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1405,6 +1405,85 @@ func TestListFormat(t *testing.T) {
}
+// noDirMoveFs wraps an Fs removing DirMove so operations.DirMove has
+// to move the objects one by one, and allows Move to be made to fail.
+type noDirMoveFs struct {
+ fs.Fs
+ features *fs.Features
+ moveCtxs chan context.Context
+}
+
+func (f *noDirMoveFs) Features() *fs.Features { return f.features }
+
+func newNoDirMoveFs(t *testing.T, wrapped fs.Fs, failOn string) *noDirMoveFs {
+ move := wrapped.Features().Move
+ require.NotNil(t, move, "the test needs a backend with Move")
+ f := &noDirMoveFs{
+ Fs: wrapped,
+ moveCtxs: make(chan context.Context, 100),
+ }
+ features := *wrapped.Features()
+ features.DirMove = nil
+ features.Move = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+ f.moveCtxs <- ctx
+ if src.Remote() == failOn {
+ return nil, errors.New("boom")
+ }
+ return move(ctx, src, remote)
+ }
+ f.features = &features
+ return f
+}
+
+// Check DirMove doesn't hang when moving the objects one by one and
+// one of the moves fails
+func TestDirMoveMoveError(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Checkers = 2
+ r := fstest.NewRun(t)
+ // More files than the renames channel can hold
+ for i := range 20 {
+ r.WriteObject(ctx, fmt.Sprintf("A/file%d", i), "x", t1)
+ }
+ f := newNoDirMoveFs(t, r.Fremote, "A/file0")
+
+ done := make(chan error, 1)
+ go func() {
+ done <- operations.DirMove(ctx, f, "A", "B")
+ }()
+ select {
+ case err := <-done:
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "boom")
+ case <-time.After(30 * time.Second):
+ t.Fatal("DirMove didn't return - deadlocked sending to the movers")
+ }
+}
+
+// Check the objects moved one by one by DirMove are moved with the
+// caller's context
+func TestDirMoveContext(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Checkers = 2
+ r := fstest.NewRun(t)
+ r.WriteObject(ctx, "A/one", "one", t1)
+ r.WriteObject(ctx, "A/two", "two", t1)
+ f := newNoDirMoveFs(t, r.Fremote, "")
+
+ require.NoError(t, operations.DirMove(accounting.WithStatsGroup(ctx, "test-dirmove"), f, "A", "B"))
+ close(f.moveCtxs)
+ moves := 0
+ for moveCtx := range f.moveCtxs {
+ group, ok := accounting.StatsGroupFromContext(moveCtx)
+ assert.True(t, ok, "move wasn't passed the caller's context")
+ assert.Equal(t, "test-dirmove", group)
+ moves++
+ }
+ assert.Equal(t, 2, moves)
+}
+
func TestDirMove(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)