"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
Add --max-connections to control maximum backend concurrency
Nick Craig-Wood267eebe5c9f3
diff --git a/fs/config.go b/fs/config.goindex 611552ffa..b9f75e2dc 100644--- a/fs/config.go+++ b/fs/config.go@@ -539,6 +539,32 @@ var ConfigOptionsInfo = Options{{Default: ".partial",Help: "Add partial-suffix to temporary file name when --inplace is not used",Groups: "Copy",+}, {+ Name: "max_connections",+ Help: strings.ReplaceAll(`Maximum number of simultaneous connections, 0 for unlimited.++This sets the maximum number of connections made to the backend on a+per backend basis. Connections in this case are calls to the backend+API and may not map 1:1 to TCP or HTTP connections depending on the+backend in use.++Note that it is possible to cause deadlocks with this setting so it+should be used with care.++If you are doing a sync or copy then make sure |--max-connections| is+one more than the sum of |--transfers| and |--checkers|.++If you use |--check-first| then |--max-connections| just needs to be+one more than the maximum of |--checkers| and |--transfers|.++So for |--max-connections 3| you'd use |--checkers 2 --transfers 2+--check-first| or |--checkers 1 --transfers 1|.++Setting this flag can be useful for backends which do multipart+uploads or downloads to limit the number of total connections.+`, "|", "`"),+ Default: 0,+ Advanced: true,}}// ConfigInfo is filesystem config options@@ -648,6 +674,7 @@ type ConfigInfo struct {Inplace bool `config:"inplace"` // Download directly to destination file instead of atomic download to temp/renamePartialSuffix string `config:"partial_suffix"`MetadataMapper SpaceSepList `config:"metadata_mapper"`+ MaxConnections int `config:"max_connections"`}func init() {diff --git a/fs/pacer.go b/fs/pacer.goindex 00515623a..f08d52154 100644--- a/fs/pacer.go+++ b/fs/pacer.go@@ -22,14 +22,12 @@ type logCalculator struct {// NewPacer creates a Pacer for the given Fs and Calculator.func NewPacer(ctx context.Context, c pacer.Calculator) *Pacer {ci := GetConfig(ctx)- retries := ci.LowLevelRetries- if retries <= 0 {- retries = 1- }+ retries := max(ci.LowLevelRetries, 1)+ maxConnections := max(ci.MaxConnections, 0)p := &Pacer{Pacer: pacer.New(pacer.InvokerOption(pacerInvoker),- // pacer.MaxConnectionsOption(ci.Checkers+ci.Transfers),+ pacer.MaxConnectionsOption(maxConnections),pacer.RetriesOption(retries),pacer.CalculatorOption(c),),