"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
drive: stop a fast-list error from hanging the sync
Sashc82efbee5b70
diff --git a/backend/drive/drive.go b/backend/drive/drive.goindex 42c563cef..22daf9571 100644--- a/backend/drive/drive.go+++ b/backend/drive/drive.go@@ -2298,15 +2298,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (wg.Wait()// if the input channel overflowed add the collected entries to the channel nowfor len(overflow) > 0 {- mu.Lock()- // only fill half of the channel to prevent entries being put into overflow again- l := min(len(overflow), listRInputBuffer/2)- wg.Add(l)- for _, d := range overflow[:l] {- in <- d+ if !refillListRInput(&mu, &in, &overflow, &wg) {+ return}- overflow = overflow[l:]- mu.Unlock()// wait again for the completion of all directorieswg.Wait()@@ -2354,6 +2348,25 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (return nil}+// refillListRInput moves queued directory jobs back onto in.+// It returns false when a listing error has already closed in.+// Sending on that nil channel would block forever while holding mu.+func refillListRInput(mu *sync.Mutex, in *chan listREntry, overflow *[]listREntry, wg *sync.WaitGroup) bool {+ mu.Lock()+ defer mu.Unlock()+ if *in == nil {+ return false+ }+ // only fill half of the channel to prevent entries being put into overflow again+ l := min(len(*overflow), listRInputBuffer/2)+ wg.Add(l)+ for _, d := range (*overflow)[:l] {+ *in <- d+ }+ *overflow = (*overflow)[l:]+ return true+}+const shortcutSeparator = '\t'// joinID adds an actual drive ID to the shortcut ID it came fromdiff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.goindex 46c727a6a..c4319abc3 100644--- a/backend/drive/drive_internal_test.go+++ b/backend/drive/drive_internal_test.go@@ -916,3 +916,51 @@ func (f *Fs) InternalTest(t *testing.T) {}var _ fstests.InternalTester = (*Fs)(nil)++func TestRefillListRInputStopsWhenInputClosed(t *testing.T) {+ mu := stdsync.Mutex{}+ wg := stdsync.WaitGroup{}+ in := make(chan listREntry, listRInputBuffer)+ overflow := []listREntry{{id: "folder", path: "folder"}}++ mu.Lock()+ close(in)+ in = nil+ mu.Unlock()++ done := make(chan bool, 1)+ go func() {+ done <- refillListRInput(&mu, &in, &overflow, &wg)+ }()+ select {+ case ok := <-done:+ require.False(t, ok, "refill continued after the input channel was closed")+ case <-time.After(time.Second):+ t.Fatal("refill blocked after the input channel was closed")+ }+ require.Len(t, overflow, 1)++ finished := make(chan struct{})+ go func() {+ wg.Wait()+ close(finished)+ }()+ select {+ case <-finished:+ case <-time.After(time.Second):+ t.Fatal("refill incremented the wait group after the input channel was closed")+ }+}++func TestRefillListRInputSendsQueuedDirectory(t *testing.T) {+ mu := stdsync.Mutex{}+ wg := stdsync.WaitGroup{}+ in := make(chan listREntry, listRInputBuffer)+ overflow := []listREntry{{id: "folder", path: "folder"}}++ require.True(t, refillListRInput(&mu, &in, &overflow, &wg), "refill stopped while the input channel was open")+ require.Empty(t, overflow)+ got := <-in+ require.Equal(t, listREntry{id: "folder", path: "folder"}, got)+ wg.Done()+}