"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs: fix goroutine leak and improve stats accounting process
Nathanael Demaconddebca8d423a
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.goindex 18249e300..f8a9980d8 100644--- a/fs/accounting/stats.go+++ b/fs/accounting/stats.go@@ -68,10 +68,11 @@ type StatsInfo struct {type averageValues struct {mu sync.Mutex+ period float64lpBytes int64lpTime time.Timespeed float64- stop chan bool+ cancel context.CancelFuncstopped sync.WaitGroupstarted bool}@@ -88,7 +89,6 @@ func NewStats(ctx context.Context) *StatsInfo {startTime: time.Now(),average: averageValues{},}- s.startAverageLoop()return s}@@ -328,84 +328,52 @@ func (s *StatsInfo) calculateTransferStats() (ts transferStats) {return ts}-func (s *StatsInfo) averageLoop() {- var period float64-+func (s *StatsInfo) averageLoop(ctx context.Context) {ticker := time.NewTicker(averagePeriodLength)defer ticker.Stop()a := &s.averagedefer a.stopped.Done()- shouldRun := false-for {select {case now := <-ticker.C:a.mu.Lock()- if !shouldRun {- a.mu.Unlock()- continue- }-avg := 0.0elapsed := now.Sub(a.lpTime).Seconds()if elapsed > 0 {avg = float64(a.lpBytes) / elapsed}- if period < averagePeriod {- period+++ if a.period < averagePeriod {+ a.period++}- a.speed = (avg + a.speed*(period-1)) / period+ a.speed = (avg + a.speed*(a.period-1)) / a.perioda.lpBytes = 0a.lpTime = nowa.mu.Unlock()- case stop, ok := <-a.stop:- if !ok {- return // Channel closed, exit the loop- }-- a.mu.Lock()-- // If we are resuming, store the current time- if !shouldRun && !stop {- a.lpTime = time.Now()- }- shouldRun = !stop-- a.mu.Unlock()+ case <-ctx.Done():+ // Stop the loop+ return}}}-// Resume the average loop-func (s *StatsInfo) resumeAverageLoop() {- s.mu.Lock()- defer s.mu.Unlock()- s.average.stop <- false-}--// Pause the average loop-func (s *StatsInfo) pauseAverageLoop() {- s.mu.Lock()- defer s.mu.Unlock()- s.average.stop <- true-}-// Start the average loop//// Call with the mutex heldfunc (s *StatsInfo) _startAverageLoop() {if !s.average.started {- s.average.stop = make(chan bool)+ ctx, cancel := context.WithCancel(s.ctx)+ s.average.cancel = cancels.average.started = trues.average.stopped.Add(1)- go s.averageLoop()+ s.average.lpTime = time.Now()+ go s.averageLoop(ctx)}}@@ -421,7 +389,7 @@ func (s *StatsInfo) startAverageLoop() {// Call with the mutex heldfunc (s *StatsInfo) _stopAverageLoop() {if s.average.started {- close(s.average.stop)+ s.average.cancel()s.average.stopped.Wait()}}@@ -839,7 +807,7 @@ func (s *StatsInfo) NewTransfer(obj fs.DirEntry, dstFs fs.Fs) *Transfer {}tr := newTransfer(s, obj, srcFs, dstFs)s.transferring.add(tr)- s.resumeAverageLoop()+ s.startAverageLoop()return tr}@@ -847,7 +815,7 @@ func (s *StatsInfo) NewTransfer(obj fs.DirEntry, dstFs fs.Fs) *Transfer {func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64, srcFs, dstFs fs.Fs) *Transfer {tr := newTransferRemoteSize(s, remote, size, false, "", srcFs, dstFs)s.transferring.add(tr)- s.resumeAverageLoop()+ s.startAverageLoop()return tr}@@ -862,7 +830,9 @@ func (s *StatsInfo) DoneTransferring(remote string, ok bool) {s.mu.Unlock()}if s.transferring.empty() && s.checking.empty() {- s.pauseAverageLoop()+ s.mu.Lock()+ s._stopAverageLoop()+ s.mu.Unlock()}}diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.goindex 3c971d831..f209d4e64 100644--- a/fs/accounting/stats_groups.go+++ b/fs/accounting/stats_groups.go@@ -298,6 +298,7 @@ func GlobalStats() *StatsInfo {// NewStatsGroup creates new stats under named group.func NewStatsGroup(ctx context.Context, group string) *StatsInfo {stats := NewStats(ctx)+ stats.startAverageLoop()stats.group = groupgroups.set(ctx, group, stats)return stats