Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: size libp2p's stream limits from the connection budget
Antonio Ojeae47aa590837e
diff --git a/internal/router/resource_limits.go b/internal/router/resource_limits.gonew file mode 100644index 00000000..8594a909--- /dev/null+++ b/internal/router/resource_limits.go@@ -0,0 +1,137 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "net/netip"+ "time"++ "github.com/google/agentmesh/api"+ "github.com/libp2p/go-libp2p"+ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/protocol"+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"+ "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"+ "github.com/libp2p/go-libp2p/p2p/protocol/identify"+ xrate "github.com/libp2p/go-libp2p/x/rate"+)++// hubLimits returns the libp2p resource limits for a router, sized from the+// connection budget rather than from host memory. libp2p's defaults are for+// a leaf peer that talks to a few dozen others and scale with the memory of+// the machine: a router on a 1 GB VM got 1024 inbound streams in all, 64 for+// identify and 64 relayed connections per peer, and refused the handshakes+// of a joining fleet (libp2p_rcmgr_blocked_resources counts them). A router+// is a hub: every member holds a connection and, on it, a few long-lived+// streams (gossip, its relay reservation) and a burst of short ones when it+// joins (identify, the auth handshake, DHT), and a service many members+// call holds one relayed connection per caller.+//+// conns is the inbound connection budget per source address and circuits+// the relayed connections one peer may hold; both come from Options.+func hubLimits(conns, circuits int) rcmgr.ConcreteLimitConfig {+ limits := rcmgr.DefaultLimits+ libp2p.SetDefaultServiceLimits(&limits)+ scaled := limits.AutoScale()++ sysBase := max(conns, DefaultHighWaterMark)+ // Streams per member on the scopes that see every member.+ const streamsPerConn = 8+ streams := func(n int) rcmgr.ResourceLimits {+ return rcmgr.ResourceLimits{+ Streams: rcmgr.LimitVal(2 * n),+ StreamsInbound: rcmgr.LimitVal(n),+ StreamsOutbound: rcmgr.LimitVal(n),+ }+ }+ system := streams(streamsPerConn * sysBase)+ system.Conns = rcmgr.LimitVal(2 * sysBase)+ system.ConnsInbound = rcmgr.LimitVal(sysBase)+ system.FD = rcmgr.LimitVal(2 * sysBase)+ transient := streams(sysBase)+ transient.Conns = rcmgr.LimitVal(sysBase)+ transient.ConnsInbound = rcmgr.LimitVal(sysBase)+ transient.FD = rcmgr.LimitVal(sysBase)++ // A relayed connection is one hop stream from the caller and one stop+ // stream to the destination, both held while it lasts; the memory the+ // relay reserves per connection is a few KiB on each.+ relayStreams := streams(sysBase)+ relayStreams.Memory = rcmgr.LimitVal64(int64(sysBase) * 64 << 10)+ perPeerCircuits := streams(circuits)++ overrides := rcmgr.PartialLimitConfig{+ System: system,+ Transient: transient,+ Service: map[string]rcmgr.ResourceLimits{+ identify.ServiceName: streams(sysBase),+ relay.ServiceName: relayStreams,+ },+ ServicePeer: map[string]rcmgr.ResourceLimits{+ relay.ServiceName: perPeerCircuits,+ },+ Protocol: map[protocol.ID]rcmgr.ResourceLimits{+ identify.ID: streams(sysBase),+ identify.IDPush: streams(sysBase),+ api.AuthProtocolID: streams(sysBase),+ circuit.ProtoIDv2Hop: relayStreams,+ circuit.ProtoIDv2Stop: relayStreams,+ },+ ProtocolPeer: map[protocol.ID]rcmgr.ResourceLimits{+ circuit.ProtoIDv2Hop: perPeerCircuits,+ circuit.ProtoIDv2Stop: perPeerCircuits,+ },+ }+ return overrides.Build(scaled)+}++// perIPConnResourceManager is libp2p's resource manager with hubLimits, the+// per-source-IP inbound connection cap and the per-subnet connection rate+// limit scaled to it; see Options.ConnsPerSourceIP. All of these matter+// behind a proxy or NAT: every peer shares a few source IPs, so libp2p's+// per-IP cap (8), its per-IP rate (0.2 conns/s, burst 16) and the small+// transient scope each take down the whole listener under normal reconnect+// churn.+func perIPConnResourceManager(limit, circuits int) (network.ResourceManager, error) {+ // Same shape as the rcmgr default limiter (loopback exempt, no global+ // cap), with the per-subnet budget scaled by limit relative to the+ // default per-IP cap of 8.+ scale := float64(limit) / 8+ connRateLimiter := &xrate.Limiter{+ NetworkPrefixLimits: []xrate.PrefixLimit{+ {Prefix: netip.MustParsePrefix("127.0.0.0/8"), Limit: xrate.Limit{}},+ {Prefix: netip.MustParsePrefix("::1/128"), Limit: xrate.Limit{}},+ },+ SubnetRateLimiter: xrate.SubnetLimiter{+ IPv4SubnetLimits: []xrate.SubnetLimit{+ {PrefixLength: 32, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},+ },+ IPv6SubnetLimits: []xrate.SubnetLimit{+ {PrefixLength: 56, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},+ },+ GracePeriod: time.Minute,+ },+ }++ return rcmgr.NewResourceManager(+ rcmgr.NewFixedLimiter(hubLimits(limit, circuits)),+ rcmgr.WithLimitPerSubnet(+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: limit}},+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: limit}},+ ),+ rcmgr.WithConnRateLimiters(connRateLimiter),+ )+}diff --git a/internal/router/resource_limits_test.go b/internal/router/resource_limits_test.gonew file mode 100644index 00000000..7c8e6898--- /dev/null+++ b/internal/router/resource_limits_test.go@@ -0,0 +1,57 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "testing"++ "github.com/google/agentmesh/api"+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"+ "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"+ "github.com/libp2p/go-libp2p/p2p/protocol/identify"+)++// TestHubLimitsFollowTheConnectionBudget pins the scopes that refused a+// joining fleet on the testnet to the connection budget instead of the+// host's memory: a router on a small VM must admit as many members as its+// watermarks say.+func TestHubLimitsFollowTheConnectionBudget(t *testing.T) {+ const conns, circuits = 6000, 1024+ cfg := hubLimits(conns, circuits).ToPartialLimitConfig()++ want := func(name string, got rcmgr.LimitVal, n int) {+ t.Helper()+ if int(got) != n {+ t.Errorf("%s = %d, want %d", name, got, n)+ }+ }+ want("system inbound conns", cfg.System.ConnsInbound, conns)+ want("system inbound streams", cfg.System.StreamsInbound, 8*conns)+ want("transient inbound streams", cfg.Transient.StreamsInbound, conns)+ want("identify service inbound streams", cfg.Service[identify.ServiceName].StreamsInbound, conns)+ want("identify protocol inbound streams", cfg.Protocol[identify.ID].StreamsInbound, conns)+ want("auth protocol inbound streams", cfg.Protocol[api.AuthProtocolID].StreamsInbound, conns)+ want("relay service inbound streams", cfg.Service[relay.ServiceName].StreamsInbound, conns)+ want("hop protocol inbound streams", cfg.Protocol[circuit.ProtoIDv2Hop].StreamsInbound, conns)+ // A destination many members call holds one stop stream per caller.+ want("relay per-peer outbound streams", cfg.ServicePeer[relay.ServiceName].StreamsOutbound, circuits)+ want("stop protocol per-peer outbound streams", cfg.ProtocolPeer[circuit.ProtoIDv2Stop].StreamsOutbound, circuits)++ // Below the default high watermark the budget is the watermark, so a+ // small per-address cap does not shrink the router.+ small := hubLimits(8, circuits).ToPartialLimitConfig()+ want("system inbound conns with a small per-address cap", small.System.ConnsInbound, DefaultHighWaterMark)+}diff --git a/internal/router/router.go b/internal/router/router.goindex c4115f68..a4c8a299 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -23,7 +23,6 @@ import ("fmt""net""net/http"- "net/netip""net/url""os""path/filepath"@@ -48,7 +47,6 @@ import ("github.com/libp2p/go-libp2p/core/host""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"- rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager""github.com/libp2p/go-libp2p/p2p/net/connmgr""github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay""github.com/libp2p/go-libp2p/p2p/security/noise"@@ -57,8 +55,7 @@ import ("github.com/libp2p/go-libp2p/p2p/transport/tcp"libp2pwebrtc "github.com/libp2p/go-libp2p/p2p/transport/webrtc"ws "github.com/libp2p/go-libp2p/p2p/transport/websocket"- "github.com/libp2p/go-libp2p/p2p/transport/webtransport"- xrate "github.com/libp2p/go-libp2p/x/rate"+ libp2pwebtransport "github.com/libp2p/go-libp2p/p2p/transport/webtransport""github.com/libp2p/go-msgio""github.com/multiformats/go-multiaddr"madns "github.com/multiformats/go-multiaddr-dns"@@ -320,62 +317,6 @@ func (r *Router) transportOptions() libp2p.Option {)}-// perIPConnResourceManager mirrors libp2p's default resource manager with the-// per-source-IP inbound connection cap, the per-subnet connection rate limit-// and the system/transient scopes scaled to carry limit connections; see-// Options.ConnsPerSourceIP. All three matter behind a proxy or NAT: every-// peer shares a few source IPs, so libp2p's per-IP cap (8), its per-IP rate-// (0.2 conns/s, burst 16) and the small transient scope each take down the-// whole listener under normal reconnect churn.-func perIPConnResourceManager(limit int) (network.ResourceManager, error) {- limits := rcmgr.DefaultLimits- libp2p.SetDefaultServiceLimits(&limits)- scaled := limits.AutoScale()-- sysBase := max(limit, DefaultHighWaterMark)- overrides := rcmgr.PartialLimitConfig{- System: rcmgr.ResourceLimits{- Conns: rcmgr.LimitVal(2 * sysBase),- ConnsInbound: rcmgr.LimitVal(sysBase),- FD: rcmgr.LimitVal(2 * sysBase),- },- Transient: rcmgr.ResourceLimits{- Conns: rcmgr.LimitVal(sysBase),- ConnsInbound: rcmgr.LimitVal(sysBase),- FD: rcmgr.LimitVal(sysBase),- },- }-- // Same shape as the rcmgr default limiter (loopback exempt, no global- // cap), with the per-subnet budget scaled by limit relative to the- // default per-IP cap of 8.- scale := float64(limit) / 8- connRateLimiter := &xrate.Limiter{- NetworkPrefixLimits: []xrate.PrefixLimit{- {Prefix: netip.MustParsePrefix("127.0.0.0/8"), Limit: xrate.Limit{}},- {Prefix: netip.MustParsePrefix("::1/128"), Limit: xrate.Limit{}},- },- SubnetRateLimiter: xrate.SubnetLimiter{- IPv4SubnetLimits: []xrate.SubnetLimit{- {PrefixLength: 32, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},- },- IPv6SubnetLimits: []xrate.SubnetLimit{- {PrefixLength: 56, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},- },- GracePeriod: time.Minute,- },- }-- return rcmgr.NewResourceManager(- rcmgr.NewFixedLimiter(overrides.Build(scaled)),- rcmgr.WithLimitPerSubnet(- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: limit}},- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: limit}},- ),- rcmgr.WithConnRateLimiters(connRateLimiter),- )-}-// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.func (r *Router) Start() (retErr error) {defer func() {@@ -460,7 +401,7 @@ func (r *Router) Start() (retErr error) {// Every refusal the resource manager makes is counted and logged with// its source; a refused dialer only sees its connection close before TLS.- mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP)+ mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits)if err != nil {return fmt.Errorf("failed to create resource manager: %w", err)}diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex 39672d87..39adfe69 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -74,6 +74,21 @@ replica count.A router stores nothing except its key. Restarting a router loses noimportant state. Nodes reconnect and publish their services again.+## Sizing++The router sizes its libp2p resource limits from `--high-watermark` and+`--conns-per-source-ip`, not from the memory of the machine: the stream+budgets of the system, of identify, of the credential handshake and of the+relay follow the connection budget, so a router on a small VM admits as+many members as its watermarks say. What the machine needs is CPU: every+member that joins costs the router a TLS handshake, an identify exchange and+a credential verification, and a router at its CPU limit answers them late+enough that members time out and retry, which costs more. Give a router at+least one full core and no CPU limit; on Kubernetes, set a request and+leave the limit off. The testnet runs its routers on 2 vCPU without a+limit, and a fleet of 500 members joining at 20 a second is admitted in+under a minute.+## MetricsThe metrics address exposes, next to the Go runtime metrics: