Skip to content

Agent Mesh

router: size libp2p's stream limits from the connection budget

Antonio Ojeae47aa590837e
diff --git a/internal/router/resource_limits.go b/internal/router/resource_limits.go
new file mode 100644
index 00000000..8594a909
--- /dev/null
+++ b/internal/router/resource_limits.go
@@ -0,0 +1,137 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "net/netip"
+ "time"
+
+ "github.com/google/agentmesh/api"
+ "github.com/libp2p/go-libp2p"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/protocol"
+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
+ "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
+ "github.com/libp2p/go-libp2p/p2p/protocol/identify"
+ xrate "github.com/libp2p/go-libp2p/x/rate"
+)
+
+// hubLimits returns the libp2p resource limits for a router, sized from the
+// connection budget rather than from host memory. libp2p's defaults are for
+// a leaf peer that talks to a few dozen others and scale with the memory of
+// the machine: a router on a 1 GB VM got 1024 inbound streams in all, 64 for
+// identify and 64 relayed connections per peer, and refused the handshakes
+// of a joining fleet (libp2p_rcmgr_blocked_resources counts them). A router
+// is a hub: every member holds a connection and, on it, a few long-lived
+// streams (gossip, its relay reservation) and a burst of short ones when it
+// joins (identify, the auth handshake, DHT), and a service many members
+// call holds one relayed connection per caller.
+//
+// conns is the inbound connection budget per source address and circuits
+// the relayed connections one peer may hold; both come from Options.
+func hubLimits(conns, circuits int) rcmgr.ConcreteLimitConfig {
+ limits := rcmgr.DefaultLimits
+ libp2p.SetDefaultServiceLimits(&limits)
+ scaled := limits.AutoScale()
+
+ sysBase := max(conns, DefaultHighWaterMark)
+ // Streams per member on the scopes that see every member.
+ const streamsPerConn = 8
+ streams := func(n int) rcmgr.ResourceLimits {
+ return rcmgr.ResourceLimits{
+ Streams: rcmgr.LimitVal(2 * n),
+ StreamsInbound: rcmgr.LimitVal(n),
+ StreamsOutbound: rcmgr.LimitVal(n),
+ }
+ }
+ system := streams(streamsPerConn * sysBase)
+ system.Conns = rcmgr.LimitVal(2 * sysBase)
+ system.ConnsInbound = rcmgr.LimitVal(sysBase)
+ system.FD = rcmgr.LimitVal(2 * sysBase)
+ transient := streams(sysBase)
+ transient.Conns = rcmgr.LimitVal(sysBase)
+ transient.ConnsInbound = rcmgr.LimitVal(sysBase)
+ transient.FD = rcmgr.LimitVal(sysBase)
+
+ // A relayed connection is one hop stream from the caller and one stop
+ // stream to the destination, both held while it lasts; the memory the
+ // relay reserves per connection is a few KiB on each.
+ relayStreams := streams(sysBase)
+ relayStreams.Memory = rcmgr.LimitVal64(int64(sysBase) * 64 << 10)
+ perPeerCircuits := streams(circuits)
+
+ overrides := rcmgr.PartialLimitConfig{
+ System: system,
+ Transient: transient,
+ Service: map[string]rcmgr.ResourceLimits{
+ identify.ServiceName: streams(sysBase),
+ relay.ServiceName: relayStreams,
+ },
+ ServicePeer: map[string]rcmgr.ResourceLimits{
+ relay.ServiceName: perPeerCircuits,
+ },
+ Protocol: map[protocol.ID]rcmgr.ResourceLimits{
+ identify.ID: streams(sysBase),
+ identify.IDPush: streams(sysBase),
+ api.AuthProtocolID: streams(sysBase),
+ circuit.ProtoIDv2Hop: relayStreams,
+ circuit.ProtoIDv2Stop: relayStreams,
+ },
+ ProtocolPeer: map[protocol.ID]rcmgr.ResourceLimits{
+ circuit.ProtoIDv2Hop: perPeerCircuits,
+ circuit.ProtoIDv2Stop: perPeerCircuits,
+ },
+ }
+ return overrides.Build(scaled)
+}
+
+// perIPConnResourceManager is libp2p's resource manager with hubLimits, the
+// per-source-IP inbound connection cap and the per-subnet connection rate
+// limit scaled to it; see Options.ConnsPerSourceIP. All of these matter
+// behind a proxy or NAT: every peer shares a few source IPs, so libp2p's
+// per-IP cap (8), its per-IP rate (0.2 conns/s, burst 16) and the small
+// transient scope each take down the whole listener under normal reconnect
+// churn.
+func perIPConnResourceManager(limit, circuits int) (network.ResourceManager, error) {
+ // Same shape as the rcmgr default limiter (loopback exempt, no global
+ // cap), with the per-subnet budget scaled by limit relative to the
+ // default per-IP cap of 8.
+ scale := float64(limit) / 8
+ connRateLimiter := &xrate.Limiter{
+ NetworkPrefixLimits: []xrate.PrefixLimit{
+ {Prefix: netip.MustParsePrefix("127.0.0.0/8"), Limit: xrate.Limit{}},
+ {Prefix: netip.MustParsePrefix("::1/128"), Limit: xrate.Limit{}},
+ },
+ SubnetRateLimiter: xrate.SubnetLimiter{
+ IPv4SubnetLimits: []xrate.SubnetLimit{
+ {PrefixLength: 32, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},
+ },
+ IPv6SubnetLimits: []xrate.SubnetLimit{
+ {PrefixLength: 56, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},
+ },
+ GracePeriod: time.Minute,
+ },
+ }
+
+ return rcmgr.NewResourceManager(
+ rcmgr.NewFixedLimiter(hubLimits(limit, circuits)),
+ rcmgr.WithLimitPerSubnet(
+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: limit}},
+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: limit}},
+ ),
+ rcmgr.WithConnRateLimiters(connRateLimiter),
+ )
+}
diff --git a/internal/router/resource_limits_test.go b/internal/router/resource_limits_test.go
new file mode 100644
index 00000000..7c8e6898
--- /dev/null
+++ b/internal/router/resource_limits_test.go
@@ -0,0 +1,57 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "testing"
+
+ "github.com/google/agentmesh/api"
+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
+ "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
+ "github.com/libp2p/go-libp2p/p2p/protocol/identify"
+)
+
+// TestHubLimitsFollowTheConnectionBudget pins the scopes that refused a
+// joining fleet on the testnet to the connection budget instead of the
+// host's memory: a router on a small VM must admit as many members as its
+// watermarks say.
+func TestHubLimitsFollowTheConnectionBudget(t *testing.T) {
+ const conns, circuits = 6000, 1024
+ cfg := hubLimits(conns, circuits).ToPartialLimitConfig()
+
+ want := func(name string, got rcmgr.LimitVal, n int) {
+ t.Helper()
+ if int(got) != n {
+ t.Errorf("%s = %d, want %d", name, got, n)
+ }
+ }
+ want("system inbound conns", cfg.System.ConnsInbound, conns)
+ want("system inbound streams", cfg.System.StreamsInbound, 8*conns)
+ want("transient inbound streams", cfg.Transient.StreamsInbound, conns)
+ want("identify service inbound streams", cfg.Service[identify.ServiceName].StreamsInbound, conns)
+ want("identify protocol inbound streams", cfg.Protocol[identify.ID].StreamsInbound, conns)
+ want("auth protocol inbound streams", cfg.Protocol[api.AuthProtocolID].StreamsInbound, conns)
+ want("relay service inbound streams", cfg.Service[relay.ServiceName].StreamsInbound, conns)
+ want("hop protocol inbound streams", cfg.Protocol[circuit.ProtoIDv2Hop].StreamsInbound, conns)
+ // A destination many members call holds one stop stream per caller.
+ want("relay per-peer outbound streams", cfg.ServicePeer[relay.ServiceName].StreamsOutbound, circuits)
+ want("stop protocol per-peer outbound streams", cfg.ProtocolPeer[circuit.ProtoIDv2Stop].StreamsOutbound, circuits)
+
+ // Below the default high watermark the budget is the watermark, so a
+ // small per-address cap does not shrink the router.
+ small := hubLimits(8, circuits).ToPartialLimitConfig()
+ want("system inbound conns with a small per-address cap", small.System.ConnsInbound, DefaultHighWaterMark)
+}
diff --git a/internal/router/router.go b/internal/router/router.go
index c4115f68..a4c8a299 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -23,7 +23,6 @@ import (
"fmt"
"net"
"net/http"
- "net/netip"
"net/url"
"os"
"path/filepath"
@@ -48,7 +47,6 @@ import (
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
- rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
"github.com/libp2p/go-libp2p/p2p/security/noise"
@@ -57,8 +55,7 @@ import (
"github.com/libp2p/go-libp2p/p2p/transport/tcp"
libp2pwebrtc "github.com/libp2p/go-libp2p/p2p/transport/webrtc"
ws "github.com/libp2p/go-libp2p/p2p/transport/websocket"
- "github.com/libp2p/go-libp2p/p2p/transport/webtransport"
- xrate "github.com/libp2p/go-libp2p/x/rate"
+ libp2pwebtransport "github.com/libp2p/go-libp2p/p2p/transport/webtransport"
"github.com/libp2p/go-msgio"
"github.com/multiformats/go-multiaddr"
madns "github.com/multiformats/go-multiaddr-dns"
@@ -320,62 +317,6 @@ func (r *Router) transportOptions() libp2p.Option {
)
}
-// perIPConnResourceManager mirrors libp2p's default resource manager with the
-// per-source-IP inbound connection cap, the per-subnet connection rate limit
-// and the system/transient scopes scaled to carry limit connections; see
-// Options.ConnsPerSourceIP. All three matter behind a proxy or NAT: every
-// peer shares a few source IPs, so libp2p's per-IP cap (8), its per-IP rate
-// (0.2 conns/s, burst 16) and the small transient scope each take down the
-// whole listener under normal reconnect churn.
-func perIPConnResourceManager(limit int) (network.ResourceManager, error) {
- limits := rcmgr.DefaultLimits
- libp2p.SetDefaultServiceLimits(&limits)
- scaled := limits.AutoScale()
-
- sysBase := max(limit, DefaultHighWaterMark)
- overrides := rcmgr.PartialLimitConfig{
- System: rcmgr.ResourceLimits{
- Conns: rcmgr.LimitVal(2 * sysBase),
- ConnsInbound: rcmgr.LimitVal(sysBase),
- FD: rcmgr.LimitVal(2 * sysBase),
- },
- Transient: rcmgr.ResourceLimits{
- Conns: rcmgr.LimitVal(sysBase),
- ConnsInbound: rcmgr.LimitVal(sysBase),
- FD: rcmgr.LimitVal(sysBase),
- },
- }
-
- // Same shape as the rcmgr default limiter (loopback exempt, no global
- // cap), with the per-subnet budget scaled by limit relative to the
- // default per-IP cap of 8.
- scale := float64(limit) / 8
- connRateLimiter := &xrate.Limiter{
- NetworkPrefixLimits: []xrate.PrefixLimit{
- {Prefix: netip.MustParsePrefix("127.0.0.0/8"), Limit: xrate.Limit{}},
- {Prefix: netip.MustParsePrefix("::1/128"), Limit: xrate.Limit{}},
- },
- SubnetRateLimiter: xrate.SubnetLimiter{
- IPv4SubnetLimits: []xrate.SubnetLimit{
- {PrefixLength: 32, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},
- },
- IPv6SubnetLimits: []xrate.SubnetLimit{
- {PrefixLength: 56, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},
- },
- GracePeriod: time.Minute,
- },
- }
-
- return rcmgr.NewResourceManager(
- rcmgr.NewFixedLimiter(overrides.Build(scaled)),
- rcmgr.WithLimitPerSubnet(
- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: limit}},
- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: limit}},
- ),
- rcmgr.WithConnRateLimiters(connRateLimiter),
- )
-}
-
// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.
func (r *Router) Start() (retErr error) {
defer func() {
@@ -460,7 +401,7 @@ func (r *Router) Start() (retErr error) {
// Every refusal the resource manager makes is counted and logged with
// its source; a refused dialer only sees its connection close before TLS.
- mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP)
+ mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits)
if err != nil {
return fmt.Errorf("failed to create resource manager: %w", err)
}
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index 39672d87..39adfe69 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -74,6 +74,21 @@ replica count.
A router stores nothing except its key. Restarting a router loses no
important state. Nodes reconnect and publish their services again.
+## Sizing
+
+The router sizes its libp2p resource limits from `--high-watermark` and
+`--conns-per-source-ip`, not from the memory of the machine: the stream
+budgets of the system, of identify, of the credential handshake and of the
+relay follow the connection budget, so a router on a small VM admits as
+many members as its watermarks say. What the machine needs is CPU: every
+member that joins costs the router a TLS handshake, an identify exchange and
+a credential verification, and a router at its CPU limit answers them late
+enough that members time out and retry, which costs more. Give a router at
+least one full core and no CPU limit; on Kubernetes, set a request and
+leave the limit off. The testnet runs its routers on 2 vCPU without a
+limit, and a fleet of 500 members joining at 20 a second is admitted in
+under a minute.
+
## Metrics
The metrics address exposes, next to the Go runtime metrics: