Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

drive: stop a fast-list error from hanging the sync

Sashc82efbee5b70
diff --git a/backend/drive/drive.go b/backend/drive/drive.go
index 42c563cef..22daf9571 100644
--- a/backend/drive/drive.go
+++ b/backend/drive/drive.go
@@ -2298,15 +2298,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
wg.Wait()
// if the input channel overflowed add the collected entries to the channel now
for len(overflow) > 0 {
- mu.Lock()
- // only fill half of the channel to prevent entries being put into overflow again
- l := min(len(overflow), listRInputBuffer/2)
- wg.Add(l)
- for _, d := range overflow[:l] {
- in <- d
+ if !refillListRInput(&mu, &in, &overflow, &wg) {
+ return
}
- overflow = overflow[l:]
- mu.Unlock()
// wait again for the completion of all directories
wg.Wait()
@@ -2354,6 +2348,25 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return nil
}
+// refillListRInput moves queued directory jobs back onto in.
+// It returns false when a listing error has already closed in.
+// Sending on that nil channel would block forever while holding mu.
+func refillListRInput(mu *sync.Mutex, in *chan listREntry, overflow *[]listREntry, wg *sync.WaitGroup) bool {
+ mu.Lock()
+ defer mu.Unlock()
+ if *in == nil {
+ return false
+ }
+ // only fill half of the channel to prevent entries being put into overflow again
+ l := min(len(*overflow), listRInputBuffer/2)
+ wg.Add(l)
+ for _, d := range (*overflow)[:l] {
+ *in <- d
+ }
+ *overflow = (*overflow)[l:]
+ return true
+}
+
const shortcutSeparator = '\t'
// joinID adds an actual drive ID to the shortcut ID it came from
diff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.go
index 46c727a6a..c4319abc3 100644
--- a/backend/drive/drive_internal_test.go
+++ b/backend/drive/drive_internal_test.go
@@ -916,3 +916,51 @@ func (f *Fs) InternalTest(t *testing.T) {
}
var _ fstests.InternalTester = (*Fs)(nil)
+
+func TestRefillListRInputStopsWhenInputClosed(t *testing.T) {
+ mu := stdsync.Mutex{}
+ wg := stdsync.WaitGroup{}
+ in := make(chan listREntry, listRInputBuffer)
+ overflow := []listREntry{{id: "folder", path: "folder"}}
+
+ mu.Lock()
+ close(in)
+ in = nil
+ mu.Unlock()
+
+ done := make(chan bool, 1)
+ go func() {
+ done <- refillListRInput(&mu, &in, &overflow, &wg)
+ }()
+ select {
+ case ok := <-done:
+ require.False(t, ok, "refill continued after the input channel was closed")
+ case <-time.After(time.Second):
+ t.Fatal("refill blocked after the input channel was closed")
+ }
+ require.Len(t, overflow, 1)
+
+ finished := make(chan struct{})
+ go func() {
+ wg.Wait()
+ close(finished)
+ }()
+ select {
+ case <-finished:
+ case <-time.After(time.Second):
+ t.Fatal("refill incremented the wait group after the input channel was closed")
+ }
+}
+
+func TestRefillListRInputSendsQueuedDirectory(t *testing.T) {
+ mu := stdsync.Mutex{}
+ wg := stdsync.WaitGroup{}
+ in := make(chan listREntry, listRInputBuffer)
+ overflow := []listREntry{{id: "folder", path: "folder"}}
+
+ require.True(t, refillListRInput(&mu, &in, &overflow, &wg), "refill stopped while the input channel was open")
+ require.Empty(t, overflow)
+ got := <-in
+ require.Equal(t, listREntry{id: "folder", path: "folder"}, got)
+ wg.Done()
+}