"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pool: add --max-buffer-memory to limit total buffer memory usage
Nick Craig-Wood19aa366d88d2
diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 4b15b37b5..c4c1de48d 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -1526,6 +1526,24 @@ of the remote which may be desirable.Setting this to a negative number will make the backlog as large aspossible.+### --max-buffer-memory=SIZE {#max-buffer-memory}++If set, don't allocate more than SIZE amount of memory as buffers. If+not set or set to `0` or `off` this will not limit the amount of memory+in use.++This includes memory used by buffers created by the `--buffer` flag+and buffers used by multi-thread transfers.++Most multi-thread transfers do not take additional memory, but some do+depending on the backend (eg the s3 backend for uploads). This means+there is a tension between total setting `--transfers` as high as+possible and memory use.++Setting `--max-buffer-memory` allows the buffer memory to be+controlled so that it doesn't overwhelm the machine and allows+`--transfers` to be set large.+### --max-delete=N ###This tells rclone not to delete more than N files. If that limit is@@ -1785,6 +1803,14 @@ This will work with the `sync`/`copy`/`move` commands and friendsmount` and `rclone serve` if `--vfs-cache-mode` is set to `writes` orabove.+Most multi-thread transfers do not take additional memory, but some do+(for example uploading to s3). In the worst case memory usage can be+at maximum `--transfers` * `--multi-thread-chunk-size` *+`--multi-thread-streams` or specifically for the s3 backend+`--transfers` * `--s3-chunk-size` * `--s3-concurrency`. However you+can use the the [--max-buffer-memory](/docs/#max-buffer-memory) flag+to control the maximum memory used here.+**NB** that this **only** works with supported backends as thedestination but will work with any backend as the source.diff --git a/docs/content/faq.md b/docs/content/faq.mdindex 050caf8ef..19eafe6dd 100644--- a/docs/content/faq.md+++ b/docs/content/faq.md@@ -239,6 +239,10 @@ memory. There is[a workaround for this](https://github.com/rclone/rclone/wiki/Big-syncs-with-millions-of-files)which involves a bit of scripting.+From v1.70 rclone also has the [--max-buffer-memory](/docs/#max-buffer-memory)+flag which helps particularly when multi-thread transfers are using+too much memory.+### Rclone changes fullwidth Unicode punctuation marks in file namesFor example: On a Windows system, you have a file with name `Test:1.jpg`,diff --git a/fs/config.go b/fs/config.goindex e83d2cac1..611552ffa 100644--- a/fs/config.go+++ b/fs/config.go@@ -413,6 +413,11 @@ var ConfigOptionsInfo = Options{{Default: false,Help: "Use mmap allocator (see docs)",Groups: "Config",+}, {+ Name: "max_buffer_memory",+ Default: SizeSuffix(-1),+ Help: "If set, don't allocate more than this amount of memory as buffers",+ Groups: "Config",}, {Name: "ca_cert",Default: []string{},@@ -613,6 +618,7 @@ type ConfigInfo struct {ProgressTerminalTitle bool `config:"progress_terminal_title"`Cookie bool `config:"use_cookies"`UseMmap bool `config:"use_mmap"`+ MaxBufferMemory SizeSuffix `config:"max_buffer_memory"`CaCert []string `config:"ca_cert"` // Client Side CAClientCert string `config:"client_cert"` // Client Side CertClientKey string `config:"client_key"` // Client Side Keydiff --git a/lib/pool/pool.go b/lib/pool/pool.goindex a6484b7cc..a4a5dc752 100644--- a/lib/pool/pool.go+++ b/lib/pool/pool.go@@ -3,12 +3,14 @@package poolimport (+ "context""fmt""sync""time""github.com/rclone/rclone/fs""github.com/rclone/rclone/lib/mmap"+ "golang.org/x/sync/semaphore")// Pool of internal buffers@@ -33,6 +35,14 @@ type Pool struct {free func([]byte) error}+// totalMemory is a semaphore used to control total buffer usage of+// all Pools. It it may be nil in which case the total buffer usage+// will not be controlled.+var totalMemory *semaphore.Weighted++// Make sure we initialise the totalMemory semaphore once+var totalMemoryInit sync.Once+// New makes a buffer pool//// flushTime is the interval the buffer pools is flushed@@ -145,6 +155,33 @@ func (bp *Pool) updateMinFill() {}}+// acquire mem bytes of memory+func (bp *Pool) acquire(mem int64) error {+ ctx := context.Background()++ totalMemoryInit.Do(func() {+ ci := fs.GetConfig(ctx)++ // Set max buffer memory limiter+ if ci.MaxBufferMemory > 0 {+ totalMemory = semaphore.NewWeighted(int64(ci.MaxBufferMemory))+ }+ })++ if totalMemory == nil {+ return nil+ }+ return totalMemory.Acquire(ctx, mem)+}++// release mem bytes of memory+func (bp *Pool) release(mem int64) {+ if totalMemory == nil {+ return+ }+ totalMemory.Release(mem)+}+// Get a buffer from the pool or allocate onefunc (bp *Pool) Get() []byte {bp.mu.Lock()@@ -156,10 +193,16 @@ func (bp *Pool) Get() []byte {break} else {var err error- buf, err = bp.alloc(bp.bufferSize)+ bp.mu.Unlock()+ err = bp.acquire(int64(bp.bufferSize))+ bp.mu.Lock()if err == nil {- bp.alloced++- break+ buf, err = bp.alloc(bp.bufferSize)+ if err == nil {+ bp.alloced+++ break+ }+ bp.release(int64(bp.bufferSize))}fs.Logf(nil, "Failed to get memory for buffer, waiting for %v: %v", waitTime, err)bp.mu.Unlock()@@ -179,6 +222,8 @@ func (bp *Pool) freeBuffer(mem []byte) {err := bp.free(mem)if err != nil {fs.Logf(nil, "Failed to free memory: %v", err)+ } else {+ bp.release(int64(bp.bufferSize))}bp.alloced--}diff --git a/lib/pool/pool_test.go b/lib/pool/pool_test.goindex 72ac42a8b..23618194a 100644--- a/lib/pool/pool_test.go+++ b/lib/pool/pool_test.go@@ -1,12 +1,15 @@package poolimport (+ "context""errors""fmt""math/rand"+ "sync""testing""time"+ "github.com/rclone/rclone/fs""github.com/rclone/rclone/fstest/testy""github.com/stretchr/testify/assert")@@ -225,3 +228,55 @@ func TestPool(t *testing.T) {})}}++func TestPoolMaxBufferMemory(t *testing.T) {+ ctx := context.Background()+ ci := fs.GetConfig(ctx)+ ci.MaxBufferMemory = 4 * 4096+ defer func() {+ ci.MaxBufferMemory = 0+ totalMemory = nil+ }()+ totalMemoryInit = sync.Once{} // reset the sync.Once as it likely has been used+ totalMemory = nil+ bp := New(60*time.Second, 4096, 2, true)++ assert.Equal(t, bp.alloced, 0)+ assert.Nil(t, totalMemory)+ buf := bp.Get()+ assert.NotNil(t, totalMemory)+ bp.Put(buf)+ assert.Equal(t, bp.alloced, 1)++ var (+ wg sync.WaitGroup+ mu sync.Mutex+ bufs int+ maxBufs int+ countBuf = func(i int) {+ mu.Lock()+ defer mu.Unlock()+ bufs += i+ if bufs > maxBufs {+ maxBufs = bufs+ }+ }+ )+ for i := 0; i < 20; i++ {+ wg.Add(1)+ go func() {+ defer wg.Done()+ buf := bp.Get()+ countBuf(1)+ time.Sleep(100 * time.Millisecond)+ bp.Put(buf)+ countBuf(-1)+ }()+ }++ wg.Wait()++ assert.Equal(t, bufs, 0)+ assert.Equal(t, maxBufs, 4)+ assert.Equal(t, bp.alloced, 2)+}