Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

accounting: add RemoveDoneTransfers method to fix bisync race #8815

Nick Craig-Wood37524e2dea2a
diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go
index b316ceb0d..1a15fe56a 100644
--- a/cmd/bisync/listing.go
+++ b/cmd/bisync/listing.go
@@ -707,8 +707,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
prettyprint(dstList.list, "dstList", fs.LogLevelDebug)
// clear stats so we only do this once
- accounting.MaxCompletedTransfers = 0
- accounting.Stats(ctx).PruneTransfers()
+ accounting.Stats(ctx).RemoveDoneTransfers()
}
if b.DebugName != "" {
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index 64f160953..6681764e5 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -912,22 +912,31 @@ func (s *StatsInfo) RemoveTransfer(transfer *Transfer) {
}
// PruneTransfers makes sure there aren't too many old transfers by removing
-// single finished transfer.
-func (s *StatsInfo) PruneTransfers() {
+// a single finished transfer. Returns true if it removed a transfer.
+func (s *StatsInfo) PruneTransfers() bool {
if MaxCompletedTransfers < 0 {
- return
+ return false
}
+ removed := false
s.mu.Lock()
// remove a transfer from the start if we are over quota
if len(s.startedTransfers) > MaxCompletedTransfers+s.ci.Transfers {
for i, tr := range s.startedTransfers {
if tr.IsDone() {
s._removeTransfer(tr, i)
+ removed = true
break
}
}
}
s.mu.Unlock()
+ return removed
+}
+
+// RemoveDoneTransfers removes all Done transfers.
+func (s *StatsInfo) RemoveDoneTransfers() {
+ for s.PruneTransfers() {
+ }
}
// AddServerSideMove counts a server side move
diff --git a/fs/accounting/stats_test.go b/fs/accounting/stats_test.go
index 6fcc09837..317cd07cd 100644
--- a/fs/accounting/stats_test.go
+++ b/fs/accounting/stats_test.go
@@ -465,3 +465,27 @@ func TestPruneTransfers(t *testing.T) {
})
}
}
+
+func TestRemoveDoneTransfers(t *testing.T) {
+ ctx := context.Background()
+ s := NewStats(ctx)
+ const transfers = 10
+ for i := int64(1); i <= int64(transfers); i++ {
+ s.AddTransfer(&Transfer{
+ startedAt: time.Unix(i, 0),
+ completedAt: time.Unix(i+1, 0),
+ })
+ }
+
+ s.mu.Lock()
+ assert.Equal(t, time.Duration(transfers)*time.Second, s._totalDuration())
+ assert.Equal(t, transfers, len(s.startedTransfers))
+ s.mu.Unlock()
+
+ s.RemoveDoneTransfers()
+
+ s.mu.Lock()
+ assert.Equal(t, time.Duration(transfers)*time.Second, s._totalDuration())
+ assert.Equal(t, transfers, len(s.startedTransfers))
+ s.mu.Unlock()
+}