Skip to content

Agent Mesh

p2plimits: charge the per-source cap to the address a connection comes from, not its relay

Antonio Ojea8a965061ddfa
diff --git a/internal/node/resource_limits.go b/internal/node/resource_limits.go
index a0d7ef21..12bef03b 100644
--- a/internal/node/resource_limits.go
+++ b/internal/node/resource_limits.go
@@ -120,12 +120,10 @@ var streamsRefusedTotal = promauto.NewCounterVec(
)
// newResourceManager is the resource manager the node's host runs with:
-// nodeLimits, the per-source cap, and every refusal counted and logged.
+// nodeLimits, the per-source cap on direct callers, and every refusal
+// counted and logged. A caller that arrives through a router is not
+// charged to the router's address; see p2plimits.Observed.
func newResourceManager() (network.ResourceManager, error) {
- mgr, err := p2plimits.NewResourceManager(nodeLimits(ConnHighWaterMark), ConnsPerSourceIP,
+ return p2plimits.New(nodeLimits(ConnHighWaterMark), ConnsPerSourceIP, inboundRefusedTotal, logger.Warnf,
rcmgr.WithMetrics(p2plimits.CountStreamRefusals(streamsRefusedTotal, logger.Warnf)))
- if err != nil {
- return nil, err
- }
- return p2plimits.Observe(mgr, inboundRefusedTotal, logger.Warnf), nil
}
diff --git a/internal/p2plimits/limits.go b/internal/p2plimits/limits.go
index 98fccf06..598a9f04 100644
--- a/internal/p2plimits/limits.go
+++ b/internal/p2plimits/limits.go
@@ -14,11 +14,13 @@
// Package p2plimits builds the libp2p resource manager a mesh component
// runs with: limits sized from its connection budget, a per-source-address
-// connection cap in place of libp2p's default of 8, and a wrapper that
-// counts and logs every inbound connection the manager refuses. libp2p
-// refuses a connection before any byte of TLS and, for its per-IP and
-// rate-limit paths, without a trace event; a dialer sees the connection
-// close or a QUIC CONNECTION_REFUSED, and nothing else records it.
+// connection cap and rate in place of libp2p's default of 8, applied to the
+// address a connection comes from and not to a relay it comes through, and
+// a wrapper that counts and logs every inbound connection the manager
+// refuses. libp2p refuses a connection before any byte of TLS and, for its
+// per-IP and rate-limit paths, without a trace event; a dialer sees the
+// connection close or a QUIC CONNECTION_REFUSED, and nothing else records
+// it.
package p2plimits
import (
@@ -50,6 +52,13 @@ const (
// Reasons lists every refusal label, for a counter to pre-register.
var Reasons = []string{RefusedPerIP, RefusedRateLimit, RefusedScope, RefusedOther}
+// What Observed answers a direct inbound connection with past the per-source
+// cap or rate; the texts are libp2p's own for the same refusals.
+var (
+ ErrPerSourceConns = errors.New("connections per ip limit exceeded")
+ ErrPerSourceRate = errors.New("rate limit exceeded")
+)
+
// ConnsPerSourceIPShare is the fraction of the inbound connection budget one
// source address may hold when no cap is configured.
const ConnsPerSourceIPShare = 4
@@ -63,25 +72,44 @@ func DefaultConnsPerSourceIP(highWaterMark int) int {
}
// libp2pDefaultConnsPerIP is the cap the resource manager applies when none
-// is configured; the per-subnet rate is scaled from it.
+// is configured; the per-source rate is scaled from it.
const libp2pDefaultConnsPerIP = 8
-// NewResourceManager is libp2p's resource manager with the given limits, a
-// per-source-IP inbound connection cap of perIP and the per-subnet
-// connection rate limit scaled to it. Loopback is exempt from both, as in
-// libp2p's default, so a local mesh never meets them. A cap under one
-// would refuse every peer, so it is an error here rather than a silent
-// listener. opts are passed on, for a MetricsReporter among others.
-func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int, opts ...rcmgr.Option) (network.ResourceManager, error) {
+// NewResourceManager is libp2p's resource manager with the given limits and
+// its own per-address limiters switched off: Observe applies the
+// per-source cap and rate, which lets it tell a connection that comes from
+// an address apart from one that comes through a relay. opts are passed
+// on, for a MetricsReporter among others.
+func NewResourceManager(limits rcmgr.ConcreteLimitConfig, opts ...rcmgr.Option) (network.ResourceManager, error) {
+ return rcmgr.NewResourceManager(
+ rcmgr.NewFixedLimiter(limits),
+ append([]rcmgr.Option{
+ // Empty, not nil: nil keeps libp2p's default of 8 per address.
+ rcmgr.WithLimitPerSubnet([]rcmgr.ConnLimitPerSubnet{}, []rcmgr.ConnLimitPerSubnet{}),
+ rcmgr.WithConnRateLimiters(&xrate.Limiter{}),
+ }, opts...)...,
+ )
+}
+
+// New is NewResourceManager wrapped in Observe: the manager a component's
+// host runs with. A cap under one would refuse every peer, so it is an
+// error here rather than a silent listener.
+func New(limits rcmgr.ConcreteLimitConfig, perIP int, refused *prometheus.CounterVec, warnf func(format string, args ...any), opts ...rcmgr.Option) (*Observed, error) {
if perIP <= 0 {
return nil, fmt.Errorf("connections per source address must be at least 1, got %d", perIP)
}
+ inner, err := NewResourceManager(limits, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return Observe(inner, perIP, refused, warnf), nil
+}
+
+// sourceRateLimiter is the per-source connection rate for a cap of perIP:
+// libp2p's default rate scaled with the cap, a burst of twice the cap.
+func sourceRateLimiter(perIP int) *xrate.Limiter {
scale := float64(perIP) / libp2pDefaultConnsPerIP
- connRateLimiter := &xrate.Limiter{
- NetworkPrefixLimits: []xrate.PrefixLimit{
- {Prefix: netip.MustParsePrefix("127.0.0.0/8"), Limit: xrate.Limit{}},
- {Prefix: netip.MustParsePrefix("::1/128"), Limit: xrate.Limit{}},
- },
+ return &xrate.Limiter{
SubnetRateLimiter: xrate.SubnetLimiter{
IPv4SubnetLimits: []xrate.SubnetLimit{
{PrefixLength: 32, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * perIP}},
@@ -92,16 +120,6 @@ func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int, opts ...rcm
GracePeriod: time.Minute,
},
}
- return rcmgr.NewResourceManager(
- rcmgr.NewFixedLimiter(limits),
- append([]rcmgr.Option{
- rcmgr.WithLimitPerSubnet(
- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: perIP}},
- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: perIP}},
- ),
- rcmgr.WithConnRateLimiters(connRateLimiter),
- }, opts...)...,
- )
}
// Streams is a stream budget of n each way.
@@ -117,29 +135,49 @@ func Streams(n int) rcmgr.ResourceLimits {
// the counter carries the rate.
const RefusalLogInterval = time.Minute
-// Observed counts and logs what the wrapped manager refuses. It keeps the
-// wrapped manager's connection limit visible to libp2p's connmgr sanity
-// check, and leaves every scope it returns untouched.
+// Observed applies the per-source cap and rate to the inbound connections
+// that come from an address, and counts and logs what it or the wrapped
+// manager refuses. A connection that comes through a relay carries the
+// relay's address, not its source's: libp2p charged it to the relay, so a
+// service called by many members through a router saw them all as one
+// address and refused most. Such a connection is bounded by the manager's
+// connection budget and its per-peer scope, and at the relay by the
+// relay's own per-peer and per-address budgets, so no per-source rule is
+// applied to it here. Loopback is exempt, as in libp2p's default, so a
+// local mesh never meets the cap. Observed keeps the wrapped manager's
+// connection limit visible to libp2p's connmgr sanity check.
type Observed struct {
network.ResourceManager
+ perIP int
+ rate *xrate.Limiter
refused *prometheus.CounterVec
warnf func(format string, args ...any)
mu sync.Mutex
+ conns map[netip.Addr]int
lastLogAt map[string]time.Time
now func() time.Time
}
-// Observe wraps inner. refused is counted by reason (see Reasons) and warnf
-// written once per source address per RefusalLogInterval; a nil warnf only
-// counts.
-func Observe(inner network.ResourceManager, refused *prometheus.CounterVec, warnf func(format string, args ...any)) *Observed {
+// Observe wraps inner with a per-source cap of perIP. refused is counted
+// by reason (see Reasons) and warnf written once per source address per
+// RefusalLogInterval; a nil warnf only counts.
+func Observe(inner network.ResourceManager, perIP int, refused *prometheus.CounterVec, warnf func(format string, args ...any)) *Observed {
for _, v := range Reasons {
refused.WithLabelValues(v)
}
if warnf == nil {
warnf = func(string, ...any) {}
}
- return &Observed{ResourceManager: inner, refused: refused, warnf: warnf, lastLogAt: map[string]time.Time{}, now: time.Now}
+ return &Observed{
+ ResourceManager: inner,
+ perIP: perIP,
+ rate: sourceRateLimiter(perIP),
+ refused: refused,
+ warnf: warnf,
+ conns: map[netip.Addr]int{},
+ lastLogAt: map[string]time.Time{},
+ now: time.Now,
+ }
}
// SetClock replaces the clock the log limiter reads; for tests.
@@ -161,11 +199,100 @@ func (m *Observed) LastLoggedAt(source string) (time.Time, bool) {
}
func (m *Observed) OpenConnection(dir network.Direction, usefd bool, endpoint multiaddr.Multiaddr) (network.ConnManagementScope, error) {
+ src, direct := sourceAddr(endpoint)
+ direct = direct && dir == network.DirInbound
+ if direct {
+ if err := m.reserve(src); err != nil {
+ m.refusedConn(endpoint, err)
+ return nil, err
+ }
+ }
scope, err := m.ResourceManager.OpenConnection(dir, usefd, endpoint)
- if err != nil && dir == network.DirInbound {
- m.refusedConn(endpoint, err)
+ if err != nil {
+ if direct {
+ m.release(src)
+ }
+ if dir == network.DirInbound {
+ m.refusedConn(endpoint, err)
+ }
+ return nil, err
+ }
+ if direct {
+ return &sourceScope{ConnManagementScope: scope, release: func() { m.release(src) }}, nil
+ }
+ return scope, nil
+}
+
+// SourceConns is how many inbound connections from addr are open.
+func (m *Observed) SourceConns(addr netip.Addr) int {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ return m.conns[addr]
+}
+
+// reserve admits one more connection from src or says why not.
+func (m *Observed) reserve(src netip.Addr) error {
+ if !m.rate.Allow(src) {
+ return ErrPerSourceRate
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.conns[src] >= m.perIP {
+ return fmt.Errorf("%w for %s", ErrPerSourceConns, src)
}
- return scope, err
+ m.conns[src]++
+ return nil
+}
+
+func (m *Observed) release(src netip.Addr) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.conns[src] <= 1 {
+ delete(m.conns, src)
+ return
+ }
+ m.conns[src]--
+}
+
+// sourceAddr is the address a connection comes from, and whether the
+// per-source rules apply to it: not to a relayed connection, whose address
+// is the relay's, nor to loopback, nor to an endpoint without an address.
+func sourceAddr(endpoint multiaddr.Multiaddr) (netip.Addr, bool) {
+ if endpoint == nil {
+ return netip.Addr{}, false
+ }
+ for _, c := range endpoint {
+ if c.Protocol().Code == multiaddr.P_CIRCUIT {
+ return netip.Addr{}, false
+ }
+ }
+ ip, err := manet.ToIP(endpoint)
+ if err != nil {
+ return netip.Addr{}, false
+ }
+ addr, ok := netip.AddrFromSlice(ip)
+ if !ok {
+ return netip.Addr{}, false
+ }
+ addr = addr.Unmap()
+ if addr.IsLoopback() {
+ return netip.Addr{}, false
+ }
+ return addr, true
+}
+
+// sourceScope gives the connection's place at its source address back when
+// the connection is done, as the wrapped manager's scope does for its own
+// books.
+type sourceScope struct {
+ network.ConnManagementScope
+ once sync.Once
+ release func()
+}
+
+func (s *sourceScope) Done() {
+ s.ConnManagementScope.Done()
+ s.once.Do(s.release)
}
// GetConnLimit is what libp2p compares the connection manager's high water
@@ -211,7 +338,12 @@ func (m *Observed) refusedConn(endpoint multiaddr.Multiaddr, err error) {
// RefusalReason maps the resource manager's errors to the counter's labels.
// The per-IP and rate-limit paths return plain strings, not sentinel errors.
func RefusalReason(err error) string {
- if errors.Is(err, network.ErrResourceLimitExceeded) {
+ switch {
+ case errors.Is(err, ErrPerSourceConns):
+ return RefusedPerIP
+ case errors.Is(err, ErrPerSourceRate):
+ return RefusedRateLimit
+ case errors.Is(err, network.ErrResourceLimitExceeded):
return RefusedScope
}
msg := err.Error()
diff --git a/internal/p2plimits/limits_test.go b/internal/p2plimits/limits_test.go
index 05dcad79..e08ec907 100644
--- a/internal/p2plimits/limits_test.go
+++ b/internal/p2plimits/limits_test.go
@@ -17,10 +17,12 @@ package p2plimits
import (
"errors"
"fmt"
+ "net/netip"
"testing"
"time"
"github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/test"
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
"github.com/multiformats/go-multiaddr"
"github.com/prometheus/client_golang/prometheus"
@@ -70,7 +72,7 @@ func TestObservedCountsAndClassifiesRefusals(t *testing.T) {
{errors.New("something else"), RefusedOther},
} {
before := counterValue(t, refused.WithLabelValues(tc.reason))
- m := Observe(&refusingResourceManager{err: tc.err}, refused, quiet)
+ m := Observe(&refusingResourceManager{err: tc.err}, 100, refused, quiet)
if _, err := m.OpenConnection(network.DirInbound, true, endpoint); err == nil {
t.Fatalf("%s: refusal not returned", tc.reason)
}
@@ -81,14 +83,14 @@ func TestObservedCountsAndClassifiesRefusals(t *testing.T) {
// An outbound failure is not an inbound refusal.
before := counterValue(t, refused.WithLabelValues(RefusedOther))
- m := Observe(&refusingResourceManager{err: errors.New("dial side")}, refused, quiet)
+ m := Observe(&refusingResourceManager{err: errors.New("dial side")}, 100, refused, quiet)
_, _ = m.OpenConnection(network.DirOutbound, true, endpoint)
if got := counterValue(t, refused.WithLabelValues(RefusedOther)); got != before {
t.Errorf("outbound failure counted as an inbound refusal")
}
// A granted connection is passed through and not counted.
- ok := Observe(&refusingResourceManager{limit: 42}, refused, quiet)
+ ok := Observe(&refusingResourceManager{limit: 42}, 100, refused, quiet)
if _, err := ok.OpenConnection(network.DirInbound, true, endpoint); err != nil {
t.Fatalf("granted connection refused: %v", err)
}
@@ -99,7 +101,7 @@ func TestObservedCountsAndClassifiesRefusals(t *testing.T) {
// The QUIC transport passes no address when it could not build one, and
// a caller may pass no logger; both are counted and nothing more.
before = counterValue(t, refused.WithLabelValues(RefusedOther))
- bare := Observe(&refusingResourceManager{err: errors.New("no address")}, refused, nil)
+ bare := Observe(&refusingResourceManager{err: errors.New("no address")}, 100, refused, nil)
if _, err := bare.OpenConnection(network.DirInbound, true, nil); err == nil {
t.Fatal("refusal with no endpoint not returned")
}
@@ -114,7 +116,7 @@ func TestObservedCountsAndClassifiesRefusals(t *testing.T) {
func TestObservedLogsOneSourceOncePerInterval(t *testing.T) {
now := time.Unix(1_700_000_000, 0)
var lines []string
- m := Observe(&refusingResourceManager{err: errors.New("rate limit exceeded")}, newRefusedCounter(),
+ m := Observe(&refusingResourceManager{err: errors.New("rate limit exceeded")}, 100, newRefusedCounter(),
func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) })
m.SetClock(func() time.Time { return now })
a := multiaddr.StringCast("/ip4/10.0.0.1/tcp/1")
@@ -145,40 +147,126 @@ func TestObservedLogsOneSourceOncePerInterval(t *testing.T) {
}
}
-// The per-source cap is what the manager enforces, and loopback is exempt.
-// A cap under one would refuse every peer and is refused itself.
-func TestNewResourceManagerCapsOneSourceAddress(t *testing.T) {
- limits := rcmgr.InfiniteLimits
- if _, err := NewResourceManager(limits, 0); err == nil {
+// The per-source cap is applied to the address a connection comes from:
+// a third direct connection from one address is refused and counted, the
+// place comes back when a connection is done, and loopback is exempt. A
+// connection that comes through a relay carries the relay's address and is
+// not charged to it: a service called by many members through one router
+// saw them all as one address. A cap under one would refuse every peer and
+// is refused itself.
+func TestObservedCapsOneSourceAddressNotItsRelay(t *testing.T) {
+ refused := newRefusedCounter()
+ if _, err := New(rcmgr.InfiniteLimits, 0, refused, nil); err == nil {
t.Fatal("a cap of 0 was accepted")
}
- m, err := NewResourceManager(limits, 2)
+ m, err := New(rcmgr.InfiniteLimits, 2, refused, nil)
if err != nil {
t.Fatal(err)
}
defer func() { _ = m.Close() }()
+ direct := multiaddr.StringCast("/ip4/203.0.113.7/tcp/1")
+ open := func(ma multiaddr.Multiaddr) (network.ConnManagementScope, error) {
+ return m.OpenConnection(network.DirInbound, true, ma)
+ }
+
var scopes []network.ConnManagementScope
for i := 0; i < 2; i++ {
- s, err := m.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/203.0.113.7/tcp/1"))
+ s, err := open(direct)
if err != nil {
t.Fatalf("connection %d from one address refused: %v", i+1, err)
}
scopes = append(scopes, s)
}
- _, err = m.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/203.0.113.7/tcp/1"))
- if err == nil || RefusalReason(err) != RefusedPerIP {
+ before := counterValue(t, refused.WithLabelValues(RefusedPerIP))
+ if _, err := open(direct); err == nil || RefusalReason(err) != RefusedPerIP {
t.Fatalf("third connection from one address: err = %v, want the per-IP refusal", err)
}
- for _, s := range scopes {
- s.Done()
+ if got := counterValue(t, refused.WithLabelValues(RefusedPerIP)); got != before+1 {
+ t.Errorf("per-IP refusals counted: %v, want %v", got, before+1)
+ }
+ if got := m.SourceConns(netip.MustParseAddr("203.0.113.7")); got != 2 {
+ t.Errorf("connections held for the address = %d, want 2", got)
+ }
+ scopes[0].Done()
+ scopes[0].Done() // a second Done gives nothing back twice
+ if got := m.SourceConns(netip.MustParseAddr("203.0.113.7")); got != 1 {
+ t.Errorf("connections held after one is done = %d, want 1", got)
+ }
+ s, err := open(direct)
+ if err != nil {
+ t.Fatalf("connection after one was done refused: %v", err)
+ }
+ s.Done()
+ scopes[1].Done()
+
+ // Through a relay: five callers, each more than the cap, and none refused.
+ relayed := multiaddr.StringCast("/ip4/203.0.113.7/tcp/1/p2p/" + test.RandPeerIDFatal(t).String() + "/p2p-circuit")
+ for i := 0; i < 5; i++ {
+ s, err := open(relayed)
+ if err != nil {
+ t.Fatalf("relayed connection %d refused: %v", i+1, err)
+ }
+ defer s.Done()
+ }
+ if got := m.SourceConns(netip.MustParseAddr("203.0.113.7")); got != 0 {
+ t.Errorf("relayed connections charged to the relay's address: %d", got)
}
for i := 0; i < 5; i++ {
- s, err := m.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/127.0.0.1/tcp/1"))
+ s, err := open(multiaddr.StringCast("/ip4/127.0.0.1/tcp/1"))
if err != nil {
t.Fatalf("loopback connection %d refused: %v", i+1, err)
}
defer s.Done()
}
+ // Outbound connections are this host's own; the cap is on what arrives.
+ for i := 0; i < 5; i++ {
+ s, err := m.OpenConnection(network.DirOutbound, true, direct)
+ if err != nil {
+ t.Fatalf("outbound connection %d refused: %v", i+1, err)
+ }
+ defer s.Done()
+ }
+}
+
+// The per-source rate follows the cap: a burst of twice the cap from one
+// address passes, the next connection in the same instant is refused as a
+// rate refusal, and the inner manager is never asked for a connection the
+// source rules refused.
+func TestObservedRateLimitsOneSourceAddress(t *testing.T) {
+ refused := newRefusedCounter()
+ inner := &countingResourceManager{}
+ m := Observe(inner, 4, refused, nil)
+ direct := multiaddr.StringCast("/ip4/198.51.100.9/udp/1/quic-v1")
+ var opened int
+ for i := 0; i < 8; i++ {
+ s, err := m.OpenConnection(network.DirInbound, true, direct)
+ if err != nil {
+ t.Fatalf("connection %d within the burst refused: %v", i+1, err)
+ }
+ s.Done()
+ opened++
+ }
+ _, err := m.OpenConnection(network.DirInbound, true, direct)
+ if err == nil || RefusalReason(err) != RefusedRateLimit {
+ t.Fatalf("connection past the burst: err = %v, want the rate refusal", err)
+ }
+ if inner.opened != opened {
+ t.Errorf("inner manager asked for %d connections, want %d: a source refusal must not reach it", inner.opened, opened)
+ }
+ if got := counterValue(t, refused.WithLabelValues(RefusedRateLimit)); got != 1 {
+ t.Errorf("rate refusals counted: %v, want 1", got)
+ }
+}
+
+// countingResourceManager grants every connection and counts them.
+type countingResourceManager struct {
+ network.NullResourceManager
+ opened int
+}
+
+func (m *countingResourceManager) OpenConnection(network.Direction, bool, multiaddr.Multiaddr) (network.ConnManagementScope, error) {
+ m.opened++
+ return &network.NullScope{}, nil
}
func TestDefaultConnsPerSourceIP(t *testing.T) {
diff --git a/internal/p2plimits/streams_test.go b/internal/p2plimits/streams_test.go
index 3514d3c4..a5af44e1 100644
--- a/internal/p2plimits/streams_test.go
+++ b/internal/p2plimits/streams_test.go
@@ -38,7 +38,7 @@ func TestStreamRefusalsCountTheProtocol(t *testing.T) {
Protocol: map[protocol.ID]rcmgr.ResourceLimits{proto: Streams(1)},
}.Build(rcmgr.InfiniteLimits)
refused := newStreamsRefusedCounter()
- m, err := NewResourceManager(limits, 8, rcmgr.WithMetrics(CountStreamRefusals(refused, nil)))
+ m, err := NewResourceManager(limits, rcmgr.WithMetrics(CountStreamRefusals(refused, nil)))
if err != nil {
t.Fatal(err)
}
diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.go
index 6242f523..a8b31b47 100644
--- a/internal/router/resource_observer.go
+++ b/internal/router/resource_observer.go
@@ -19,7 +19,6 @@ import (
"time"
"github.com/google/agentmesh/internal/p2plimits"
- "github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@@ -50,12 +49,6 @@ func init() {
}
}
-// observeResourceManager counts the wrapped manager's refusals in
-// inboundRefusedTotal and logs them, once per source per minute.
-func observeResourceManager(inner network.ResourceManager) *p2plimits.Observed {
- return p2plimits.Observe(inner, inboundRefusedTotal, logger.Warnf)
-}
-
// countStreamRefusals is the reporter the resource manager is built with;
// it counts refused streams in streamsRefusedTotal by protocol.
func countStreamRefusals() *p2plimits.StreamRefusals {
diff --git a/internal/router/router.go b/internal/router/router.go
index 492a0f4c..167f9551 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -490,11 +490,11 @@ func (r *Router) Start() (retErr error) {
// Every refusal the resource manager makes is counted and logged with
// its source; a refused dialer only sees its connection close before TLS.
- mgr, err := p2plimits.NewResourceManager(hubLimits(r.config.HighWaterMark, r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits), r.config.ConnsPerSourceIP, rcmgr.WithMetrics(countStreamRefusals()))
+ mgr, err := p2plimits.New(hubLimits(r.config.HighWaterMark, r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits), r.config.ConnsPerSourceIP, inboundRefusedTotal, logger.Warnf, rcmgr.WithMetrics(countStreamRefusals()))
if err != nil {
return fmt.Errorf("failed to create resource manager: %w", err)
}
- p2pOpts = append(p2pOpts, libp2p.ResourceManager(observeResourceManager(mgr)))
+ p2pOpts = append(p2pOpts, libp2p.ResourceManager(mgr))
hostNode, err := libp2p.New(p2pOpts...)
if err != nil {
diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.md
index 760b2366..707bdb5c 100644
--- a/site/content/docs/guides/scalability.md
+++ b/site/content/docs/guides/scalability.md
@@ -50,7 +50,7 @@ them.
| Relayed connections per peer | `agentmesh-router --relay-max-circuits` | 1024 | A service called through the relay holds one circuit per caller; the next caller is refused. |
| Enrollments per second | control plane, fixed | 10 a second, burst 20 | The next enrollment is answered `429` with `Retry-After`; the node retries for up to three minutes. |
| Connections per node | `agentmesh-node`, fixed | 400 high, 100 low | A node called by more peers than this at once closes connections down to 100; past 400 it refuses the next one. |
-| Connections per source address at a node | `agentmesh-node`, fixed | a quarter of the high mark (100) | The node refuses the next connection from that address, counted in `agentmesh_node_inbound_connections_refused_total`. A service called by a fleet behind one NAT or one cluster's egress sees them all from one address. |
+| Connections per source address at a node | `agentmesh-node`, fixed | a quarter of the high mark (100) | The node refuses the next direct connection from that address, counted in `agentmesh_node_inbound_connections_refused_total`. A service called by a fleet behind one NAT or one cluster's egress sees them all from one address. A caller that arrives through a router is not charged to the router's address: it is bounded by the node's connection budget and per-peer scope, and at the router by the relay's per-peer and per-address budgets. |
| Router lease | `agentmesh-control-plane --lease-duration`, `agentmesh-router --shutdown-lease-ttl` | 15m, 30s | A router that stops without the last lease stays listed for the lease duration. |
The first two rows are the model. A fleet of F members on N routers, each
@@ -234,7 +234,7 @@ The router and the control plane serve Prometheus metrics on
| `agentmesh_router_connections{direction="inbound"}` against `agentmesh_router_connection_watermark{level="high"}` | How full a router is. The gap, summed over the routers that are not draining, is the free capacity of the mesh. |
| `agentmesh_router_goaway_sent_total{reason="OVERLOADED"}` | A router at its high mark sent members to the others. Rising on a router that is not draining, the fleet has outgrown its routers. |
| `agentmesh_router_inbound_connections_refused_total` | Members behind one address have hit `--conns-per-source-ip`. |
-| `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `scope_limit` more callers at once than the node's connection budget. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. |
+| `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: direct callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `rate_limit` a burst from one address, `scope_limit` more callers at once than the node's connection budget, relayed ones included. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. |
| `agentmesh_node_transit_handshakes_total{router,result}` | On a caller: handshakes with routers it did not hold, run to reach a peer through them. `router="unlisted"` is a router the node's catalog missed, which a member that started during a router rollout produces until its next sync; `result="failed"` rising is a router that refuses the node or is gone while peers still announce circuits through it. |
| `agentmesh_node_peer_lookups_total{result}` | On a caller: DHT lookups for a peer it held no address for. `not_found` is a peer the DHT does not carry, and the call falls back to circuits through the caller's own routers, which reach the peer only if it holds one of them; `failed` rising is the caller cut off from the DHT (`agentmesh_node_dht_routing_table_size` at 0). |
| `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. |