Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

Add --max-connections to control maximum backend concurrency

Nick Craig-Wood267eebe5c9f3
diff --git a/fs/config.go b/fs/config.go
index 611552ffa..b9f75e2dc 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -539,6 +539,32 @@ var ConfigOptionsInfo = Options{{
Default: ".partial",
Help: "Add partial-suffix to temporary file name when --inplace is not used",
Groups: "Copy",
+}, {
+ Name: "max_connections",
+ Help: strings.ReplaceAll(`Maximum number of simultaneous connections, 0 for unlimited.
+
+This sets the maximum number of connections made to the backend on a
+per backend basis. Connections in this case are calls to the backend
+API and may not map 1:1 to TCP or HTTP connections depending on the
+backend in use.
+
+Note that it is possible to cause deadlocks with this setting so it
+should be used with care.
+
+If you are doing a sync or copy then make sure |--max-connections| is
+one more than the sum of |--transfers| and |--checkers|.
+
+If you use |--check-first| then |--max-connections| just needs to be
+one more than the maximum of |--checkers| and |--transfers|.
+
+So for |--max-connections 3| you'd use |--checkers 2 --transfers 2
+--check-first| or |--checkers 1 --transfers 1|.
+
+Setting this flag can be useful for backends which do multipart
+uploads or downloads to limit the number of total connections.
+`, "|", "`"),
+ Default: 0,
+ Advanced: true,
}}
// ConfigInfo is filesystem config options
@@ -648,6 +674,7 @@ type ConfigInfo struct {
Inplace bool `config:"inplace"` // Download directly to destination file instead of atomic download to temp/rename
PartialSuffix string `config:"partial_suffix"`
MetadataMapper SpaceSepList `config:"metadata_mapper"`
+ MaxConnections int `config:"max_connections"`
}
func init() {
diff --git a/fs/pacer.go b/fs/pacer.go
index 00515623a..f08d52154 100644
--- a/fs/pacer.go
+++ b/fs/pacer.go
@@ -22,14 +22,12 @@ type logCalculator struct {
// NewPacer creates a Pacer for the given Fs and Calculator.
func NewPacer(ctx context.Context, c pacer.Calculator) *Pacer {
ci := GetConfig(ctx)
- retries := ci.LowLevelRetries
- if retries <= 0 {
- retries = 1
- }
+ retries := max(ci.LowLevelRetries, 1)
+ maxConnections := max(ci.MaxConnections, 0)
p := &Pacer{
Pacer: pacer.New(
pacer.InvokerOption(pacerInvoker),
- // pacer.MaxConnectionsOption(ci.Checkers+ci.Transfers),
+ pacer.MaxConnectionsOption(maxConnections),
pacer.RetriesOption(retries),
pacer.CalculatorOption(c),
),