Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
p2plimits: charge the per-source cap to the address a connection comes from, not its relay
Antonio Ojea8a965061ddfa
diff --git a/internal/node/resource_limits.go b/internal/node/resource_limits.goindex a0d7ef21..12bef03b 100644--- a/internal/node/resource_limits.go+++ b/internal/node/resource_limits.go@@ -120,12 +120,10 @@ var streamsRefusedTotal = promauto.NewCounterVec()// newResourceManager is the resource manager the node's host runs with:-// nodeLimits, the per-source cap, and every refusal counted and logged.+// nodeLimits, the per-source cap on direct callers, and every refusal+// counted and logged. A caller that arrives through a router is not+// charged to the router's address; see p2plimits.Observed.func newResourceManager() (network.ResourceManager, error) {- mgr, err := p2plimits.NewResourceManager(nodeLimits(ConnHighWaterMark), ConnsPerSourceIP,+ return p2plimits.New(nodeLimits(ConnHighWaterMark), ConnsPerSourceIP, inboundRefusedTotal, logger.Warnf,rcmgr.WithMetrics(p2plimits.CountStreamRefusals(streamsRefusedTotal, logger.Warnf)))- if err != nil {- return nil, err- }- return p2plimits.Observe(mgr, inboundRefusedTotal, logger.Warnf), nil}diff --git a/internal/p2plimits/limits.go b/internal/p2plimits/limits.goindex 98fccf06..598a9f04 100644--- a/internal/p2plimits/limits.go+++ b/internal/p2plimits/limits.go@@ -14,11 +14,13 @@// Package p2plimits builds the libp2p resource manager a mesh component// runs with: limits sized from its connection budget, a per-source-address-// connection cap in place of libp2p's default of 8, and a wrapper that-// counts and logs every inbound connection the manager refuses. libp2p-// refuses a connection before any byte of TLS and, for its per-IP and-// rate-limit paths, without a trace event; a dialer sees the connection-// close or a QUIC CONNECTION_REFUSED, and nothing else records it.+// connection cap and rate in place of libp2p's default of 8, applied to the+// address a connection comes from and not to a relay it comes through, and+// a wrapper that counts and logs every inbound connection the manager+// refuses. libp2p refuses a connection before any byte of TLS and, for its+// per-IP and rate-limit paths, without a trace event; a dialer sees the+// connection close or a QUIC CONNECTION_REFUSED, and nothing else records+// it.package p2plimitsimport (@@ -50,6 +52,13 @@ const (// Reasons lists every refusal label, for a counter to pre-register.var Reasons = []string{RefusedPerIP, RefusedRateLimit, RefusedScope, RefusedOther}+// What Observed answers a direct inbound connection with past the per-source+// cap or rate; the texts are libp2p's own for the same refusals.+var (+ ErrPerSourceConns = errors.New("connections per ip limit exceeded")+ ErrPerSourceRate = errors.New("rate limit exceeded")+)+// ConnsPerSourceIPShare is the fraction of the inbound connection budget one// source address may hold when no cap is configured.const ConnsPerSourceIPShare = 4@@ -63,25 +72,44 @@ func DefaultConnsPerSourceIP(highWaterMark int) int {}// libp2pDefaultConnsPerIP is the cap the resource manager applies when none-// is configured; the per-subnet rate is scaled from it.+// is configured; the per-source rate is scaled from it.const libp2pDefaultConnsPerIP = 8-// NewResourceManager is libp2p's resource manager with the given limits, a-// per-source-IP inbound connection cap of perIP and the per-subnet-// connection rate limit scaled to it. Loopback is exempt from both, as in-// libp2p's default, so a local mesh never meets them. A cap under one-// would refuse every peer, so it is an error here rather than a silent-// listener. opts are passed on, for a MetricsReporter among others.-func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int, opts ...rcmgr.Option) (network.ResourceManager, error) {+// NewResourceManager is libp2p's resource manager with the given limits and+// its own per-address limiters switched off: Observe applies the+// per-source cap and rate, which lets it tell a connection that comes from+// an address apart from one that comes through a relay. opts are passed+// on, for a MetricsReporter among others.+func NewResourceManager(limits rcmgr.ConcreteLimitConfig, opts ...rcmgr.Option) (network.ResourceManager, error) {+ return rcmgr.NewResourceManager(+ rcmgr.NewFixedLimiter(limits),+ append([]rcmgr.Option{+ // Empty, not nil: nil keeps libp2p's default of 8 per address.+ rcmgr.WithLimitPerSubnet([]rcmgr.ConnLimitPerSubnet{}, []rcmgr.ConnLimitPerSubnet{}),+ rcmgr.WithConnRateLimiters(&xrate.Limiter{}),+ }, opts...)...,+ )+}++// New is NewResourceManager wrapped in Observe: the manager a component's+// host runs with. A cap under one would refuse every peer, so it is an+// error here rather than a silent listener.+func New(limits rcmgr.ConcreteLimitConfig, perIP int, refused *prometheus.CounterVec, warnf func(format string, args ...any), opts ...rcmgr.Option) (*Observed, error) {if perIP <= 0 {return nil, fmt.Errorf("connections per source address must be at least 1, got %d", perIP)}+ inner, err := NewResourceManager(limits, opts...)+ if err != nil {+ return nil, err+ }+ return Observe(inner, perIP, refused, warnf), nil+}++// sourceRateLimiter is the per-source connection rate for a cap of perIP:+// libp2p's default rate scaled with the cap, a burst of twice the cap.+func sourceRateLimiter(perIP int) *xrate.Limiter {scale := float64(perIP) / libp2pDefaultConnsPerIP- connRateLimiter := &xrate.Limiter{- NetworkPrefixLimits: []xrate.PrefixLimit{- {Prefix: netip.MustParsePrefix("127.0.0.0/8"), Limit: xrate.Limit{}},- {Prefix: netip.MustParsePrefix("::1/128"), Limit: xrate.Limit{}},- },+ return &xrate.Limiter{SubnetRateLimiter: xrate.SubnetLimiter{IPv4SubnetLimits: []xrate.SubnetLimit{{PrefixLength: 32, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * perIP}},@@ -92,16 +120,6 @@ func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int, opts ...rcmGracePeriod: time.Minute,},}- return rcmgr.NewResourceManager(- rcmgr.NewFixedLimiter(limits),- append([]rcmgr.Option{- rcmgr.WithLimitPerSubnet(- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: perIP}},- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: perIP}},- ),- rcmgr.WithConnRateLimiters(connRateLimiter),- }, opts...)...,- )}// Streams is a stream budget of n each way.@@ -117,29 +135,49 @@ func Streams(n int) rcmgr.ResourceLimits {// the counter carries the rate.const RefusalLogInterval = time.Minute-// Observed counts and logs what the wrapped manager refuses. It keeps the-// wrapped manager's connection limit visible to libp2p's connmgr sanity-// check, and leaves every scope it returns untouched.+// Observed applies the per-source cap and rate to the inbound connections+// that come from an address, and counts and logs what it or the wrapped+// manager refuses. A connection that comes through a relay carries the+// relay's address, not its source's: libp2p charged it to the relay, so a+// service called by many members through a router saw them all as one+// address and refused most. Such a connection is bounded by the manager's+// connection budget and its per-peer scope, and at the relay by the+// relay's own per-peer and per-address budgets, so no per-source rule is+// applied to it here. Loopback is exempt, as in libp2p's default, so a+// local mesh never meets the cap. Observed keeps the wrapped manager's+// connection limit visible to libp2p's connmgr sanity check.type Observed struct {network.ResourceManager+ perIP int+ rate *xrate.Limiterrefused *prometheus.CounterVecwarnf func(format string, args ...any)mu sync.Mutex+ conns map[netip.Addr]intlastLogAt map[string]time.Timenow func() time.Time}-// Observe wraps inner. refused is counted by reason (see Reasons) and warnf-// written once per source address per RefusalLogInterval; a nil warnf only-// counts.-func Observe(inner network.ResourceManager, refused *prometheus.CounterVec, warnf func(format string, args ...any)) *Observed {+// Observe wraps inner with a per-source cap of perIP. refused is counted+// by reason (see Reasons) and warnf written once per source address per+// RefusalLogInterval; a nil warnf only counts.+func Observe(inner network.ResourceManager, perIP int, refused *prometheus.CounterVec, warnf func(format string, args ...any)) *Observed {for _, v := range Reasons {refused.WithLabelValues(v)}if warnf == nil {warnf = func(string, ...any) {}}- return &Observed{ResourceManager: inner, refused: refused, warnf: warnf, lastLogAt: map[string]time.Time{}, now: time.Now}+ return &Observed{+ ResourceManager: inner,+ perIP: perIP,+ rate: sourceRateLimiter(perIP),+ refused: refused,+ warnf: warnf,+ conns: map[netip.Addr]int{},+ lastLogAt: map[string]time.Time{},+ now: time.Now,+ }}// SetClock replaces the clock the log limiter reads; for tests.@@ -161,11 +199,100 @@ func (m *Observed) LastLoggedAt(source string) (time.Time, bool) {}func (m *Observed) OpenConnection(dir network.Direction, usefd bool, endpoint multiaddr.Multiaddr) (network.ConnManagementScope, error) {+ src, direct := sourceAddr(endpoint)+ direct = direct && dir == network.DirInbound+ if direct {+ if err := m.reserve(src); err != nil {+ m.refusedConn(endpoint, err)+ return nil, err+ }+ }scope, err := m.ResourceManager.OpenConnection(dir, usefd, endpoint)- if err != nil && dir == network.DirInbound {- m.refusedConn(endpoint, err)+ if err != nil {+ if direct {+ m.release(src)+ }+ if dir == network.DirInbound {+ m.refusedConn(endpoint, err)+ }+ return nil, err+ }+ if direct {+ return &sourceScope{ConnManagementScope: scope, release: func() { m.release(src) }}, nil+ }+ return scope, nil+}++// SourceConns is how many inbound connections from addr are open.+func (m *Observed) SourceConns(addr netip.Addr) int {+ m.mu.Lock()+ defer m.mu.Unlock()+ return m.conns[addr]+}++// reserve admits one more connection from src or says why not.+func (m *Observed) reserve(src netip.Addr) error {+ if !m.rate.Allow(src) {+ return ErrPerSourceRate+ }+ m.mu.Lock()+ defer m.mu.Unlock()+ if m.conns[src] >= m.perIP {+ return fmt.Errorf("%w for %s", ErrPerSourceConns, src)}- return scope, err+ m.conns[src]+++ return nil+}++func (m *Observed) release(src netip.Addr) {+ m.mu.Lock()+ defer m.mu.Unlock()+ if m.conns[src] <= 1 {+ delete(m.conns, src)+ return+ }+ m.conns[src]--+}++// sourceAddr is the address a connection comes from, and whether the+// per-source rules apply to it: not to a relayed connection, whose address+// is the relay's, nor to loopback, nor to an endpoint without an address.+func sourceAddr(endpoint multiaddr.Multiaddr) (netip.Addr, bool) {+ if endpoint == nil {+ return netip.Addr{}, false+ }+ for _, c := range endpoint {+ if c.Protocol().Code == multiaddr.P_CIRCUIT {+ return netip.Addr{}, false+ }+ }+ ip, err := manet.ToIP(endpoint)+ if err != nil {+ return netip.Addr{}, false+ }+ addr, ok := netip.AddrFromSlice(ip)+ if !ok {+ return netip.Addr{}, false+ }+ addr = addr.Unmap()+ if addr.IsLoopback() {+ return netip.Addr{}, false+ }+ return addr, true+}++// sourceScope gives the connection's place at its source address back when+// the connection is done, as the wrapped manager's scope does for its own+// books.+type sourceScope struct {+ network.ConnManagementScope+ once sync.Once+ release func()+}++func (s *sourceScope) Done() {+ s.ConnManagementScope.Done()+ s.once.Do(s.release)}// GetConnLimit is what libp2p compares the connection manager's high water@@ -211,7 +338,12 @@ func (m *Observed) refusedConn(endpoint multiaddr.Multiaddr, err error) {// RefusalReason maps the resource manager's errors to the counter's labels.// The per-IP and rate-limit paths return plain strings, not sentinel errors.func RefusalReason(err error) string {- if errors.Is(err, network.ErrResourceLimitExceeded) {+ switch {+ case errors.Is(err, ErrPerSourceConns):+ return RefusedPerIP+ case errors.Is(err, ErrPerSourceRate):+ return RefusedRateLimit+ case errors.Is(err, network.ErrResourceLimitExceeded):return RefusedScope}msg := err.Error()diff --git a/internal/p2plimits/limits_test.go b/internal/p2plimits/limits_test.goindex 05dcad79..e08ec907 100644--- a/internal/p2plimits/limits_test.go+++ b/internal/p2plimits/limits_test.go@@ -17,10 +17,12 @@ package p2plimitsimport ("errors""fmt"+ "net/netip""testing""time""github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/test"rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager""github.com/multiformats/go-multiaddr""github.com/prometheus/client_golang/prometheus"@@ -70,7 +72,7 @@ func TestObservedCountsAndClassifiesRefusals(t *testing.T) {{errors.New("something else"), RefusedOther},} {before := counterValue(t, refused.WithLabelValues(tc.reason))- m := Observe(&refusingResourceManager{err: tc.err}, refused, quiet)+ m := Observe(&refusingResourceManager{err: tc.err}, 100, refused, quiet)if _, err := m.OpenConnection(network.DirInbound, true, endpoint); err == nil {t.Fatalf("%s: refusal not returned", tc.reason)}@@ -81,14 +83,14 @@ func TestObservedCountsAndClassifiesRefusals(t *testing.T) {// An outbound failure is not an inbound refusal.before := counterValue(t, refused.WithLabelValues(RefusedOther))- m := Observe(&refusingResourceManager{err: errors.New("dial side")}, refused, quiet)+ m := Observe(&refusingResourceManager{err: errors.New("dial side")}, 100, refused, quiet)_, _ = m.OpenConnection(network.DirOutbound, true, endpoint)if got := counterValue(t, refused.WithLabelValues(RefusedOther)); got != before {t.Errorf("outbound failure counted as an inbound refusal")}// A granted connection is passed through and not counted.- ok := Observe(&refusingResourceManager{limit: 42}, refused, quiet)+ ok := Observe(&refusingResourceManager{limit: 42}, 100, refused, quiet)if _, err := ok.OpenConnection(network.DirInbound, true, endpoint); err != nil {t.Fatalf("granted connection refused: %v", err)}@@ -99,7 +101,7 @@ func TestObservedCountsAndClassifiesRefusals(t *testing.T) {// The QUIC transport passes no address when it could not build one, and// a caller may pass no logger; both are counted and nothing more.before = counterValue(t, refused.WithLabelValues(RefusedOther))- bare := Observe(&refusingResourceManager{err: errors.New("no address")}, refused, nil)+ bare := Observe(&refusingResourceManager{err: errors.New("no address")}, 100, refused, nil)if _, err := bare.OpenConnection(network.DirInbound, true, nil); err == nil {t.Fatal("refusal with no endpoint not returned")}@@ -114,7 +116,7 @@ func TestObservedCountsAndClassifiesRefusals(t *testing.T) {func TestObservedLogsOneSourceOncePerInterval(t *testing.T) {now := time.Unix(1_700_000_000, 0)var lines []string- m := Observe(&refusingResourceManager{err: errors.New("rate limit exceeded")}, newRefusedCounter(),+ m := Observe(&refusingResourceManager{err: errors.New("rate limit exceeded")}, 100, newRefusedCounter(),func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) })m.SetClock(func() time.Time { return now })a := multiaddr.StringCast("/ip4/10.0.0.1/tcp/1")@@ -145,40 +147,126 @@ func TestObservedLogsOneSourceOncePerInterval(t *testing.T) {}}-// The per-source cap is what the manager enforces, and loopback is exempt.-// A cap under one would refuse every peer and is refused itself.-func TestNewResourceManagerCapsOneSourceAddress(t *testing.T) {- limits := rcmgr.InfiniteLimits- if _, err := NewResourceManager(limits, 0); err == nil {+// The per-source cap is applied to the address a connection comes from:+// a third direct connection from one address is refused and counted, the+// place comes back when a connection is done, and loopback is exempt. A+// connection that comes through a relay carries the relay's address and is+// not charged to it: a service called by many members through one router+// saw them all as one address. A cap under one would refuse every peer and+// is refused itself.+func TestObservedCapsOneSourceAddressNotItsRelay(t *testing.T) {+ refused := newRefusedCounter()+ if _, err := New(rcmgr.InfiniteLimits, 0, refused, nil); err == nil {t.Fatal("a cap of 0 was accepted")}- m, err := NewResourceManager(limits, 2)+ m, err := New(rcmgr.InfiniteLimits, 2, refused, nil)if err != nil {t.Fatal(err)}defer func() { _ = m.Close() }()+ direct := multiaddr.StringCast("/ip4/203.0.113.7/tcp/1")+ open := func(ma multiaddr.Multiaddr) (network.ConnManagementScope, error) {+ return m.OpenConnection(network.DirInbound, true, ma)+ }+var scopes []network.ConnManagementScopefor i := 0; i < 2; i++ {- s, err := m.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/203.0.113.7/tcp/1"))+ s, err := open(direct)if err != nil {t.Fatalf("connection %d from one address refused: %v", i+1, err)}scopes = append(scopes, s)}- _, err = m.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/203.0.113.7/tcp/1"))- if err == nil || RefusalReason(err) != RefusedPerIP {+ before := counterValue(t, refused.WithLabelValues(RefusedPerIP))+ if _, err := open(direct); err == nil || RefusalReason(err) != RefusedPerIP {t.Fatalf("third connection from one address: err = %v, want the per-IP refusal", err)}- for _, s := range scopes {- s.Done()+ if got := counterValue(t, refused.WithLabelValues(RefusedPerIP)); got != before+1 {+ t.Errorf("per-IP refusals counted: %v, want %v", got, before+1)+ }+ if got := m.SourceConns(netip.MustParseAddr("203.0.113.7")); got != 2 {+ t.Errorf("connections held for the address = %d, want 2", got)+ }+ scopes[0].Done()+ scopes[0].Done() // a second Done gives nothing back twice+ if got := m.SourceConns(netip.MustParseAddr("203.0.113.7")); got != 1 {+ t.Errorf("connections held after one is done = %d, want 1", got)+ }+ s, err := open(direct)+ if err != nil {+ t.Fatalf("connection after one was done refused: %v", err)+ }+ s.Done()+ scopes[1].Done()++ // Through a relay: five callers, each more than the cap, and none refused.+ relayed := multiaddr.StringCast("/ip4/203.0.113.7/tcp/1/p2p/" + test.RandPeerIDFatal(t).String() + "/p2p-circuit")+ for i := 0; i < 5; i++ {+ s, err := open(relayed)+ if err != nil {+ t.Fatalf("relayed connection %d refused: %v", i+1, err)+ }+ defer s.Done()+ }+ if got := m.SourceConns(netip.MustParseAddr("203.0.113.7")); got != 0 {+ t.Errorf("relayed connections charged to the relay's address: %d", got)}for i := 0; i < 5; i++ {- s, err := m.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/127.0.0.1/tcp/1"))+ s, err := open(multiaddr.StringCast("/ip4/127.0.0.1/tcp/1"))if err != nil {t.Fatalf("loopback connection %d refused: %v", i+1, err)}defer s.Done()}+ // Outbound connections are this host's own; the cap is on what arrives.+ for i := 0; i < 5; i++ {+ s, err := m.OpenConnection(network.DirOutbound, true, direct)+ if err != nil {+ t.Fatalf("outbound connection %d refused: %v", i+1, err)+ }+ defer s.Done()+ }+}++// The per-source rate follows the cap: a burst of twice the cap from one+// address passes, the next connection in the same instant is refused as a+// rate refusal, and the inner manager is never asked for a connection the+// source rules refused.+func TestObservedRateLimitsOneSourceAddress(t *testing.T) {+ refused := newRefusedCounter()+ inner := &countingResourceManager{}+ m := Observe(inner, 4, refused, nil)+ direct := multiaddr.StringCast("/ip4/198.51.100.9/udp/1/quic-v1")+ var opened int+ for i := 0; i < 8; i++ {+ s, err := m.OpenConnection(network.DirInbound, true, direct)+ if err != nil {+ t.Fatalf("connection %d within the burst refused: %v", i+1, err)+ }+ s.Done()+ opened+++ }+ _, err := m.OpenConnection(network.DirInbound, true, direct)+ if err == nil || RefusalReason(err) != RefusedRateLimit {+ t.Fatalf("connection past the burst: err = %v, want the rate refusal", err)+ }+ if inner.opened != opened {+ t.Errorf("inner manager asked for %d connections, want %d: a source refusal must not reach it", inner.opened, opened)+ }+ if got := counterValue(t, refused.WithLabelValues(RefusedRateLimit)); got != 1 {+ t.Errorf("rate refusals counted: %v, want 1", got)+ }+}++// countingResourceManager grants every connection and counts them.+type countingResourceManager struct {+ network.NullResourceManager+ opened int+}++func (m *countingResourceManager) OpenConnection(network.Direction, bool, multiaddr.Multiaddr) (network.ConnManagementScope, error) {+ m.opened+++ return &network.NullScope{}, nil}func TestDefaultConnsPerSourceIP(t *testing.T) {diff --git a/internal/p2plimits/streams_test.go b/internal/p2plimits/streams_test.goindex 3514d3c4..a5af44e1 100644--- a/internal/p2plimits/streams_test.go+++ b/internal/p2plimits/streams_test.go@@ -38,7 +38,7 @@ func TestStreamRefusalsCountTheProtocol(t *testing.T) {Protocol: map[protocol.ID]rcmgr.ResourceLimits{proto: Streams(1)},}.Build(rcmgr.InfiniteLimits)refused := newStreamsRefusedCounter()- m, err := NewResourceManager(limits, 8, rcmgr.WithMetrics(CountStreamRefusals(refused, nil)))+ m, err := NewResourceManager(limits, rcmgr.WithMetrics(CountStreamRefusals(refused, nil)))if err != nil {t.Fatal(err)}diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.goindex 6242f523..a8b31b47 100644--- a/internal/router/resource_observer.go+++ b/internal/router/resource_observer.go@@ -19,7 +19,6 @@ import ("time""github.com/google/agentmesh/internal/p2plimits"- "github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer""github.com/prometheus/client_golang/prometheus""github.com/prometheus/client_golang/prometheus/promauto"@@ -50,12 +49,6 @@ func init() {}}-// observeResourceManager counts the wrapped manager's refusals in-// inboundRefusedTotal and logs them, once per source per minute.-func observeResourceManager(inner network.ResourceManager) *p2plimits.Observed {- return p2plimits.Observe(inner, inboundRefusedTotal, logger.Warnf)-}-// countStreamRefusals is the reporter the resource manager is built with;// it counts refused streams in streamsRefusedTotal by protocol.func countStreamRefusals() *p2plimits.StreamRefusals {diff --git a/internal/router/router.go b/internal/router/router.goindex 492a0f4c..167f9551 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -490,11 +490,11 @@ func (r *Router) Start() (retErr error) {// Every refusal the resource manager makes is counted and logged with// its source; a refused dialer only sees its connection close before TLS.- mgr, err := p2plimits.NewResourceManager(hubLimits(r.config.HighWaterMark, r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits), r.config.ConnsPerSourceIP, rcmgr.WithMetrics(countStreamRefusals()))+ mgr, err := p2plimits.New(hubLimits(r.config.HighWaterMark, r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits), r.config.ConnsPerSourceIP, inboundRefusedTotal, logger.Warnf, rcmgr.WithMetrics(countStreamRefusals()))if err != nil {return fmt.Errorf("failed to create resource manager: %w", err)}- p2pOpts = append(p2pOpts, libp2p.ResourceManager(observeResourceManager(mgr)))+ p2pOpts = append(p2pOpts, libp2p.ResourceManager(mgr))hostNode, err := libp2p.New(p2pOpts...)if err != nil {diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.mdindex 760b2366..707bdb5c 100644--- a/site/content/docs/guides/scalability.md+++ b/site/content/docs/guides/scalability.md@@ -50,7 +50,7 @@ them.| Relayed connections per peer | `agentmesh-router --relay-max-circuits` | 1024 | A service called through the relay holds one circuit per caller; the next caller is refused. || Enrollments per second | control plane, fixed | 10 a second, burst 20 | The next enrollment is answered `429` with `Retry-After`; the node retries for up to three minutes. || Connections per node | `agentmesh-node`, fixed | 400 high, 100 low | A node called by more peers than this at once closes connections down to 100; past 400 it refuses the next one. |-| Connections per source address at a node | `agentmesh-node`, fixed | a quarter of the high mark (100) | The node refuses the next connection from that address, counted in `agentmesh_node_inbound_connections_refused_total`. A service called by a fleet behind one NAT or one cluster's egress sees them all from one address. |+| Connections per source address at a node | `agentmesh-node`, fixed | a quarter of the high mark (100) | The node refuses the next direct connection from that address, counted in `agentmesh_node_inbound_connections_refused_total`. A service called by a fleet behind one NAT or one cluster's egress sees them all from one address. A caller that arrives through a router is not charged to the router's address: it is bounded by the node's connection budget and per-peer scope, and at the router by the relay's per-peer and per-address budgets. || Router lease | `agentmesh-control-plane --lease-duration`, `agentmesh-router --shutdown-lease-ttl` | 15m, 30s | A router that stops without the last lease stays listed for the lease duration. |The first two rows are the model. A fleet of F members on N routers, each@@ -234,7 +234,7 @@ The router and the control plane serve Prometheus metrics on| `agentmesh_router_connections{direction="inbound"}` against `agentmesh_router_connection_watermark{level="high"}` | How full a router is. The gap, summed over the routers that are not draining, is the free capacity of the mesh. || `agentmesh_router_goaway_sent_total{reason="OVERLOADED"}` | A router at its high mark sent members to the others. Rising on a router that is not draining, the fleet has outgrown its routers. || `agentmesh_router_inbound_connections_refused_total` | Members behind one address have hit `--conns-per-source-ip`. |-| `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `scope_limit` more callers at once than the node's connection budget. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. |+| `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: direct callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `rate_limit` a burst from one address, `scope_limit` more callers at once than the node's connection budget, relayed ones included. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. || `agentmesh_node_transit_handshakes_total{router,result}` | On a caller: handshakes with routers it did not hold, run to reach a peer through them. `router="unlisted"` is a router the node's catalog missed, which a member that started during a router rollout produces until its next sync; `result="failed"` rising is a router that refuses the node or is gone while peers still announce circuits through it. || `agentmesh_node_peer_lookups_total{result}` | On a caller: DHT lookups for a peer it held no address for. `not_found` is a peer the DHT does not carry, and the call falls back to circuits through the caller's own routers, which reach the peer only if it holds one of them; `failed` rising is the caller cut off from the DHT (`agentmesh_node_dht_routing_table_size` at 0). || `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. |