"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
accounting: add RemoveDoneTransfers method to fix bisync race #8815
Nick Craig-Wood37524e2dea2a
diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.goindex b316ceb0d..1a15fe56a 100644--- a/cmd/bisync/listing.go+++ b/cmd/bisync/listing.go@@ -707,8 +707,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, resprettyprint(dstList.list, "dstList", fs.LogLevelDebug)// clear stats so we only do this once- accounting.MaxCompletedTransfers = 0- accounting.Stats(ctx).PruneTransfers()+ accounting.Stats(ctx).RemoveDoneTransfers()}if b.DebugName != "" {diff --git a/fs/accounting/stats.go b/fs/accounting/stats.goindex 64f160953..6681764e5 100644--- a/fs/accounting/stats.go+++ b/fs/accounting/stats.go@@ -912,22 +912,31 @@ func (s *StatsInfo) RemoveTransfer(transfer *Transfer) {}// PruneTransfers makes sure there aren't too many old transfers by removing-// single finished transfer.-func (s *StatsInfo) PruneTransfers() {+// a single finished transfer. Returns true if it removed a transfer.+func (s *StatsInfo) PruneTransfers() bool {if MaxCompletedTransfers < 0 {- return+ return false}+ removed := falses.mu.Lock()// remove a transfer from the start if we are over quotaif len(s.startedTransfers) > MaxCompletedTransfers+s.ci.Transfers {for i, tr := range s.startedTransfers {if tr.IsDone() {s._removeTransfer(tr, i)+ removed = truebreak}}}s.mu.Unlock()+ return removed+}++// RemoveDoneTransfers removes all Done transfers.+func (s *StatsInfo) RemoveDoneTransfers() {+ for s.PruneTransfers() {+ }}// AddServerSideMove counts a server side movediff --git a/fs/accounting/stats_test.go b/fs/accounting/stats_test.goindex 6fcc09837..317cd07cd 100644--- a/fs/accounting/stats_test.go+++ b/fs/accounting/stats_test.go@@ -465,3 +465,27 @@ func TestPruneTransfers(t *testing.T) {})}}++func TestRemoveDoneTransfers(t *testing.T) {+ ctx := context.Background()+ s := NewStats(ctx)+ const transfers = 10+ for i := int64(1); i <= int64(transfers); i++ {+ s.AddTransfer(&Transfer{+ startedAt: time.Unix(i, 0),+ completedAt: time.Unix(i+1, 0),+ })+ }++ s.mu.Lock()+ assert.Equal(t, time.Duration(transfers)*time.Second, s._totalDuration())+ assert.Equal(t, transfers, len(s.startedTransfers))+ s.mu.Unlock()++ s.RemoveDoneTransfers()++ s.mu.Lock()+ assert.Equal(t, time.Duration(transfers)*time.Second, s._totalDuration())+ assert.Equal(t, transfers, len(s.startedTransfers))+ s.mu.Unlock()+}