"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: add method to real time account server side copy
Nick Craig-Wood307f1edaf4b0
diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.goindex 5e5a3138b..0fd9af92f 100644--- a/fs/accounting/accounting.go+++ b/fs/accounting/accounting.go@@ -11,6 +11,7 @@ import ("unicode/utf8""github.com/rclone/rclone/fs/rc"+ "github.com/rclone/rclone/lib/transferaccounter""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/asyncreader"@@ -312,6 +313,15 @@ func (acc *Account) serverSideEnd(n int64) {}}+// NewServerSideCopyAccounter returns a TransferAccounter for a server+// side copy and a new ctx with it embedded+func (acc *Account) NewServerSideCopyAccounter(ctx context.Context) (context.Context, *transferaccounter.TransferAccounter) {+ return transferaccounter.New(ctx, func(n int64) {+ acc.stats.AddServerSideCopyBytes(n)+ acc.accountReadNoNetwork(n)+ })+}+// ServerSideCopyEnd accounts for a read of n bytes in a server-side copyfunc (acc *Account) ServerSideCopyEnd(n int64) {acc.stats.AddServerSideCopy(n)@@ -363,6 +373,17 @@ func (acc *Account) accountRead(n int) {acc.limitPerFileBandwidth(n)}+// Account the read if not using network (eg for server side copies)+func (acc *Account) accountReadNoNetwork(n int64) {+ // Update Stats+ acc.values.mu.Lock()+ acc.values.lpBytes += n+ acc.values.bytes += n+ acc.values.mu.Unlock()++ acc.stats.BytesNoNetwork(n)+}+// read bytes from the io.Reader passed in and account themfunc (acc *Account) read(in io.Reader, p []byte) (n int, err error) {bytesUntilLimit, err := acc.checkReadBefore()diff --git a/fs/accounting/stats.go b/fs/accounting/stats.goindex fbc46bebb..41d0462e3 100644--- a/fs/accounting/stats.go+++ b/fs/accounting/stats.go@@ -960,6 +960,13 @@ func (s *StatsInfo) AddServerSideMove(n int64) {s.mu.Unlock()}+// AddServerSideCopyBytes adds bytes for a server side copy+func (s *StatsInfo) AddServerSideCopyBytes(n int64) {+ s.mu.Lock()+ s.serverSideCopyBytes += n+ s.mu.Unlock()+}+// AddServerSideCopy counts a server side copyfunc (s *StatsInfo) AddServerSideCopy(n int64) {s.mu.Lock()diff --git a/fs/operations/copy.go b/fs/operations/copy.goindex f7871d292..094aa288c 100644--- a/fs/operations/copy.go+++ b/fs/operations/copy.go@@ -151,9 +151,17 @@ func (c *copy) serverSideCopy(ctx context.Context) (actionTaken string, newDst f}in := c.tr.Account(ctx, nil) // account the transferin.ServerSideTransferStart()- newDst, err = doCopy(ctx, c.src, c.remoteForCopy)+ newCtx, ta := in.NewServerSideCopyAccounter(ctx)+ newDst, err = doCopy(newCtx, c.src, c.remoteForCopy)if err == nil {- in.ServerSideCopyEnd(newDst.Size()) // account the bytes for the server-side transfer+ var n int64+ if !ta.Started() {+ n = newDst.Size()+ }+ in.ServerSideCopyEnd(n) // account the bytes for the server-side transfer+ } else {+ // Rewind any stats counted on error+ ta.Reset()}_ = in.Close()if errors.Is(err, fs.ErrorCantCopy) {diff --git a/lib/transferaccounter/transferaccounter.go b/lib/transferaccounter/transferaccounter.gonew file mode 100644index 000000000..fdf5ae9c6--- /dev/null+++ b/lib/transferaccounter/transferaccounter.go@@ -0,0 +1,75 @@+// Package transferaccounter provides utilities for accounting server side transfers.+package transferaccounter++import (+ "context"+ "sync/atomic"+)++// Context key type for accounter+type accounterContextKeyType struct{}++// Context key for accounter+var accounterContextKey = accounterContextKeyType{}++// TransferAccounter is used to account server side and other transfers.+type TransferAccounter struct {+ add func(n int64)+ total atomic.Int64+ started bool+}++// New creates a TransferAccounter using the add function passed in.+//+// Note that the add function should be goroutine safe.+//+// It adds the new TransferAccounter to the context.+func New(ctx context.Context, add func(n int64)) (context.Context, *TransferAccounter) {+ ta := &TransferAccounter{+ add: add,+ }+ newCtx := context.WithValue(ctx, accounterContextKey, ta)+ return newCtx, ta+}++// Start the transfer. Call this before calling Add().+func (ta *TransferAccounter) Start() {+ ta.started = true+}++// Started returns if the transfer has had Start() called or not.+func (ta *TransferAccounter) Started() bool {+ return ta.started+}++// Add n bytes to the transfer+func (ta *TransferAccounter) Add(n int64) {+ ta.add(n)+ ta.total.Add(n)+}++// Reset reverses out all accounted stats if Started() has been called+func (ta *TransferAccounter) Reset() {+ if ta.started {+ ta.Add(-ta.total.Load())+ }+}++// A transfer accounter which does nothing+var nullAccounter = &TransferAccounter{+ add: func(n int64) {},+}++// Get returns a *TransferAccounter from the ctx.+//+// If none is found it will return a dummy one to keep the code simple.+func Get(ctx context.Context) *TransferAccounter {+ if ctx == nil {+ return nullAccounter+ }+ c := ctx.Value(accounterContextKey)+ if c == nil {+ return nullAccounter+ }+ return c.(*TransferAccounter)+}diff --git a/lib/transferaccounter/transferaccounter_test.go b/lib/transferaccounter/transferaccounter_test.gonew file mode 100644index 000000000..a60fb788f--- /dev/null+++ b/lib/transferaccounter/transferaccounter_test.go@@ -0,0 +1,91 @@+package transferaccounter++import (+ "context"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestNew(t *testing.T) {+ // Dummy add function+ var totalBytes int64+ addFn := func(n int64) {+ totalBytes += n+ }++ // Create the accounter+ ctx := context.Background()+ _, ta := New(ctx, addFn)++ // Verify object creation+ require.NotNil(t, ta)+ assert.False(t, ta.Started(), "New accounter should not be started by default")++ // Test Start()+ ta.Start()+ assert.True(t, ta.Started(), "Accounter should be started after calling Start()")++ // Test Add() logic+ ta.Add(100)+ ta.Add(50)+ assert.Equal(t, int64(150), totalBytes, "The add function should have been called with cumulative values")+ assert.Equal(t, int64(150), ta.total.Load(), "Internal counter did not count")++ // Test Reset() logic+ ta.Reset()+ assert.Equal(t, int64(0), totalBytes, "The Reset function failed")+ assert.Equal(t, int64(0), ta.total.Load(), "Internal counter did not reset")++}++func TestGet(t *testing.T) {+ t.Run("Retrieve existing accounter", func(t *testing.T) {+ // Create a specific accounter to identify later+ expectedTotal := int64(0)+ ctx, originalTa := New(context.Background(), func(n int64) { expectedTotal += n })++ // Retrieve it+ retrievedTa := Get(ctx)++ // Assert it is the exact same pointer+ assert.Equal(t, originalTa, retrievedTa)++ // Verify functionality passes through+ retrievedTa.Add(10)+ assert.Equal(t, int64(10), expectedTotal)+ })++ t.Run("Context does not contain accounter", func(t *testing.T) {+ ctx := context.Background()+ ta := Get(ctx)++ assert.NotNil(t, ta, "Get should never return nil")+ assert.Equal(t, nullAccounter, ta, "Should return the global nullAccounter")+ })++ t.Run("Context is nil", func(t *testing.T) {+ ta := Get(nil) //nolint:staticcheck // we want to test this++ assert.NotNil(t, ta, "Get should never return nil")+ assert.Equal(t, nullAccounter, ta, "Should return the global nullAccounter")+ })+}++func TestNullAccounterBehavior(t *testing.T) {+ // Ensure the null accounter (returned when context is missing/nil)+ // can be called without panicking.+ ta := Get(nil) //nolint:staticcheck // we want to test this++ assert.NotPanics(t, func() {+ ta.Start()+ })++ // Even after start, it acts as a valid object+ assert.True(t, ta.Started())++ assert.NotPanics(t, func() {+ ta.Add(1000)+ })+}