"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pacer: factor call stack searching into its own package
Nick Craig-Woodb5e4d39b054c
diff --git a/lib/caller/caller.go b/lib/caller/caller.gonew file mode 100644index 000000000..9004d36d8--- /dev/null+++ b/lib/caller/caller.go@@ -0,0 +1,26 @@+// Package caller contains functions to examine the call stack.+package caller++import (+ "runtime"+ "strings"+)++// Present looks for functionName in the call stack and return true if found+//+// Note that this ignores the caller.+func Present(functionName string) bool {+ var pcs [48]uintptr+ n := runtime.Callers(3, pcs[:]) // skip runtime.Callers, Present and caller+ frames := runtime.CallersFrames(pcs[:n])+ for {+ f, more := frames.Next()+ if strings.HasSuffix(f.Function, functionName) {+ return true+ }+ if !more {+ break+ }+ }+ return false+}diff --git a/lib/caller/caller_test.go b/lib/caller/caller_test.gonew file mode 100644index 000000000..e3d13e6ab--- /dev/null+++ b/lib/caller/caller_test.go@@ -0,0 +1,37 @@+package caller++import (+ "testing"++ "github.com/stretchr/testify/assert"+)++func TestPresent(t *testing.T) {+ assert.False(t, Present("NotFound"))+ assert.False(t, Present("TestPresent"))+ f := func() {+ assert.True(t, Present("TestPresent"))+ }+ f()+}++func BenchmarkPresent(b *testing.B) {+ for b.Loop() {+ _ = Present("NotFound")+ }+}++func BenchmarkPresent100(b *testing.B) {+ var fn func(level int)+ fn = func(level int) {+ if level > 0 {+ fn(level - 1)+ return+ }+ for b.Loop() {+ _ = Present("NotFound")+ }++ }+ fn(100)+}diff --git a/lib/pacer/pacer.go b/lib/pacer/pacer.goindex 005317821..357ff7d49 100644--- a/lib/pacer/pacer.go+++ b/lib/pacer/pacer.go@@ -4,11 +4,10 @@ package pacerimport ("errors""fmt"- "runtime"- "strings""sync""time"+ "github.com/rclone/rclone/lib/caller"liberrors "github.com/rclone/rclone/lib/errors")@@ -193,7 +192,9 @@ func (p *Pacer) endCall(retry bool, err error, limitConnections bool) {p.mu.Unlock()}-// Detect the pacer being called reentrantly.+// call implements Call but with settable retries+//+// This detects the pacer being called reentrantly.//// This looks for Pacer.call in the call stack and returns true if it// is found.@@ -204,27 +205,10 @@ func (p *Pacer) endCall(retry bool, err error, limitConnections bool) {// This is only needed when p.maxConnections > 0 which isn't a common// configuration so adding a bit of extra slowdown here is not a// problem.-func pacerReentered() bool {- var pcs [48]uintptr- n := runtime.Callers(3, pcs[:]) // skip runtime.Callers, pacerReentered and call- frames := runtime.CallersFrames(pcs[:n])- for {- f, more := frames.Next()- if strings.HasSuffix(f.Function, "(*Pacer).call") {- return true- }- if !more {- break- }- }- return false-}--// call implements Call but with settable retriesfunc (p *Pacer) call(fn Paced, retries int) (err error) {var retry boollimitConnections := false- if p.maxConnections > 0 && !pacerReentered() {+ if p.maxConnections > 0 && !caller.Present("(*Pacer).call") {limitConnections = true}for i := 1; i <= retries; i++ {diff --git a/lib/pacer/pacer_test.go b/lib/pacer/pacer_test.goindex 76ef4b071..5c5bbd298 100644--- a/lib/pacer/pacer_test.go+++ b/lib/pacer/pacer_test.go@@ -353,27 +353,6 @@ func TestCallParallel(t *testing.T) {wait.Broadcast()}-func BenchmarkPacerReentered(b *testing.B) {- for b.Loop() {- _ = pacerReentered()- }-}--func BenchmarkPacerReentered100(b *testing.B) {- var fn func(level int)- fn = func(level int) {- if level > 0 {- fn(level - 1)- return- }- for b.Loop() {- _ = pacerReentered()- }-- }- fn(100)-}-func TestCallMaxConnectionsRecursiveDeadlock(t *testing.T) {p := New(CalculatorOption(NewDefault(MinSleep(1*time.Millisecond), MaxSleep(2*time.Millisecond))))p.SetMaxConnections(1)