Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router, node: budget the DHT and gossip streams per member, count refused streams by protocol
Antonio Ojea9c453b903e37
diff --git a/api/network.go b/api/network.goindex be60a092..02a5a458 100644--- a/api/network.go+++ b/api/network.go@@ -44,6 +44,12 @@ const (// it will no longer hold it; the stream carries one RouterGoAway.GoAwayProtocolID protocol.ID = "/mesh/goaway/1.0.0"+ // DHTProtocolPrefix namespaces the mesh's Kademlia DHT away from the+ // public IPFS one; the protocol every member and router registers for+ // it is DHTProtocolID.+ DHTProtocolPrefix protocol.ID = "/mesh"+ DHTProtocolID protocol.ID = DHTProtocolPrefix + "/kad/1.0.0"+// GossipEvents is the GossipSub topic used to broadcast mesh event updates (e.g., node bans).GossipEvents = "/mesh/events/v1"diff --git a/internal/node/node.go b/internal/node/node.goindex 3caf6228..5e6eb174 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -689,7 +689,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {// Initialize Rendezvous (DHT Client)dhtOpts := []dht.Option{dht.Mode(dht.ModeAuto),- dht.ProtocolPrefix("/mesh"),+ dht.ProtocolPrefix(api.DHTProtocolPrefix),}providerTTL := n.config.DHTProviderAddrTTLif providerTTL <= 0 {diff --git a/internal/node/resource_limits.go b/internal/node/resource_limits.goindex 7c6fff47..a0d7ef21 100644--- a/internal/node/resource_limits.go+++ b/internal/node/resource_limits.go@@ -15,9 +15,12 @@package nodeimport (+ "slices"+"github.com/google/agentmesh/api""github.com/google/agentmesh/internal/p2plimits""github.com/libp2p/go-libp2p"+ pubsub "github.com/libp2p/go-libp2p-pubsub""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/protocol"rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"@@ -91,19 +94,36 @@ func nodeLimits(conns int) rcmgr.ConcreteLimitConfig {Service: map[string]rcmgr.ResourceLimits{identify.ServiceName: p2plimits.Streams(conns),},- Protocol: map[protocol.ID]rcmgr.ResourceLimits{- identify.ID: p2plimits.Streams(conns),- identify.IDPush: p2plimits.Streams(conns),- api.AuthProtocolID: p2plimits.Streams(conns),- },+ Protocol: map[protocol.ID]rcmgr.ResourceLimits{},}+ // One stream per caller for as long as it is connected (its DHT request+ // stream, gossip), or one when it arrives; libp2p's default for a+ // protocol it was not told about scales from host memory. A caller holds+ // one MCP stream per session, so that protocol gets the per-connection+ // share of the system budget.+ for _, p := range slices.Concat(+ []protocol.ID{identify.ID, identify.IDPush, api.AuthProtocolID, api.DHTProtocolID},+ pubsub.GossipSubDefaultProtocols,+ ) {+ overrides.Protocol[p] = p2plimits.Streams(conns)+ }+ overrides.Protocol[api.MCPProtocolID] = p2plimits.Streams(streamsPerConn * conns)return overrides.Build(scaled)}+var streamsRefusedTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "agentmesh_node_streams_refused_total",+ Help: "Streams the resource manager refused, by scope and protocol (or service)",+ },+ []string{"scope", "protocol"},+)+// newResourceManager is the resource manager the node's host runs with:// nodeLimits, the per-source cap, and every refusal counted and logged.func newResourceManager() (network.ResourceManager, error) {- mgr, err := p2plimits.NewResourceManager(nodeLimits(ConnHighWaterMark), ConnsPerSourceIP)+ mgr, err := p2plimits.NewResourceManager(nodeLimits(ConnHighWaterMark), ConnsPerSourceIP,+ rcmgr.WithMetrics(p2plimits.CountStreamRefusals(streamsRefusedTotal, logger.Warnf)))if err != nil {return nil, err}diff --git a/internal/node/resource_limits_test.go b/internal/node/resource_limits_test.goindex a43ebc98..5542233b 100644--- a/internal/node/resource_limits_test.go+++ b/internal/node/resource_limits_test.go@@ -18,6 +18,7 @@ import ("testing""github.com/google/agentmesh/api"+ pubsub "github.com/libp2p/go-libp2p-pubsub""github.com/libp2p/go-libp2p/core/connmgr"rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager""github.com/libp2p/go-libp2p/p2p/protocol/identify"@@ -39,6 +40,12 @@ func TestNodeLimitsFollowTheConnectionBudget(t *testing.T) {want("transient inbound conns", cfg.Transient.ConnsInbound, ConnHighWaterMark)want("identify service inbound streams", cfg.Service[identify.ServiceName].StreamsInbound, ConnHighWaterMark)want("auth protocol inbound streams", cfg.Protocol[api.AuthProtocolID].StreamsInbound, ConnHighWaterMark)+ // A caller holds its MCP, DHT and gossip streams for as long as it stays.+ want("mcp protocol inbound streams", cfg.Protocol[api.MCPProtocolID].StreamsInbound, 8*ConnHighWaterMark)+ want("dht protocol inbound streams", cfg.Protocol[api.DHTProtocolID].StreamsInbound, ConnHighWaterMark)+ for _, p := range pubsub.GossipSubDefaultProtocols {+ want("gossip protocol "+string(p)+" inbound streams", cfg.Protocol[p].StreamsInbound, ConnHighWaterMark)+ }if ConnsPerSourceIP != ConnHighWaterMark/4 {t.Errorf("ConnsPerSourceIP = %d, want a quarter of the high watermark, %d", ConnsPerSourceIP, ConnHighWaterMark/4)diff --git a/internal/p2plimits/limits.go b/internal/p2plimits/limits.goindex dccdb0ac..98fccf06 100644--- a/internal/p2plimits/limits.go+++ b/internal/p2plimits/limits.go@@ -71,8 +71,8 @@ const libp2pDefaultConnsPerIP = 8// connection rate limit scaled to it. Loopback is exempt from both, as in// libp2p's default, so a local mesh never meets them. A cap under one// would refuse every peer, so it is an error here rather than a silent-// listener.-func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int) (network.ResourceManager, error) {+// listener. opts are passed on, for a MetricsReporter among others.+func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int, opts ...rcmgr.Option) (network.ResourceManager, error) {if perIP <= 0 {return nil, fmt.Errorf("connections per source address must be at least 1, got %d", perIP)}@@ -94,11 +94,13 @@ func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int) (network.Re}return rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(limits),- rcmgr.WithLimitPerSubnet(- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: perIP}},- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: perIP}},- ),- rcmgr.WithConnRateLimiters(connRateLimiter),+ append([]rcmgr.Option{+ rcmgr.WithLimitPerSubnet(+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: perIP}},+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: perIP}},+ ),+ rcmgr.WithConnRateLimiters(connRateLimiter),+ }, opts...)...,)}diff --git a/internal/p2plimits/streams.go b/internal/p2plimits/streams.gonew file mode 100644index 00000000..7e7e124e--- /dev/null+++ b/internal/p2plimits/streams.go@@ -0,0 +1,123 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package p2plimits++import (+ "sync"+ "time"++ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-libp2p/core/protocol"+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"+ "github.com/prometheus/client_golang/prometheus"+)++// The scopes a stream refusal is counted under. libp2p's own gauge+// (libp2p_rcmgr_blocked_resources) carries the scope but not the protocol,+// and a hub needs the protocol to tell a flood from a limit set too low.+const (+ StreamScopeProtocol = "protocol"+ StreamScopeProtocolPeer = "protocol_peer"+ StreamScopeService = "service"+ StreamScopeServicePeer = "service_peer"+ StreamScopePeer = "peer"+ // StreamScopeConn is the system or transient scope, where the manager+ // does not yet know the protocol.+ StreamScopeConn = "conn"+)++// StreamRefusals counts the streams the resource manager refused, by scope+// and protocol (or service), and logs each scope and name once per+// RefusalLogInterval. It is libp2p's MetricsReporter; the Allow side is a+// no-op.+type StreamRefusals struct {+ refused *prometheus.CounterVec+ warnf func(format string, args ...any)+ mu sync.Mutex+ since map[string]uint64+ lastLogAt map[string]time.Time+ now func() time.Time+}++var _ rcmgr.MetricsReporter = (*StreamRefusals)(nil)++// CountStreamRefusals returns a reporter for rcmgr.WithMetrics. refused+// carries the labels scope and protocol; a nil warnf only counts.+func CountStreamRefusals(refused *prometheus.CounterVec, warnf func(format string, args ...any)) *StreamRefusals {+ if warnf == nil {+ warnf = func(string, ...any) {}+ }+ return &StreamRefusals{+ refused: refused,+ warnf: warnf,+ since: map[string]uint64{},+ lastLogAt: map[string]time.Time{},+ now: time.Now,+ }+}++// SetClock replaces the clock the log interval is measured on.+func (s *StreamRefusals) SetClock(now func() time.Time) { s.now = now }++func (s *StreamRefusals) refusedStream(scope, name string) {+ s.refused.WithLabelValues(scope, name).Inc()+ key := scope + " " + name+ s.mu.Lock()+ s.since[key]+++ n := s.since[key]+ now := s.now()+ last, seen := s.lastLogAt[key]+ shouldLog := !seen || now.Sub(last) >= RefusalLogInterval+ if shouldLog {+ s.lastLogAt[key] = now+ s.since[key] = 0+ }+ s.mu.Unlock()+ if shouldLog {+ s.warnf("[Stream] Refused %d stream(s) on %s at the %s scope since the last report; the peer sees its stream reset", n, name, scope)+ }+}++func (s *StreamRefusals) BlockStream(_ peer.ID, _ network.Direction) {+ s.refusedStream(StreamScopeConn, "")+}++func (s *StreamRefusals) BlockPeer(peer.ID) { s.refusedStream(StreamScopePeer, "") }++func (s *StreamRefusals) BlockProtocol(proto protocol.ID) {+ s.refusedStream(StreamScopeProtocol, string(proto))+}++func (s *StreamRefusals) BlockProtocolPeer(proto protocol.ID, _ peer.ID) {+ s.refusedStream(StreamScopeProtocolPeer, string(proto))+}++func (s *StreamRefusals) BlockService(svc string) { s.refusedStream(StreamScopeService, svc) }++func (s *StreamRefusals) BlockServicePeer(svc string, _ peer.ID) {+ s.refusedStream(StreamScopeServicePeer, svc)+}++// Connections and memory are observed elsewhere (Observed and libp2p's own+// gauges); allowances are not of interest.+func (s *StreamRefusals) AllowConn(network.Direction, bool) {}+func (s *StreamRefusals) BlockConn(network.Direction, bool) {}+func (s *StreamRefusals) AllowStream(peer.ID, network.Direction) {}+func (s *StreamRefusals) AllowPeer(peer.ID) {}+func (s *StreamRefusals) AllowProtocol(protocol.ID) {}+func (s *StreamRefusals) AllowService(string) {}+func (s *StreamRefusals) AllowMemory(int) {}+func (s *StreamRefusals) BlockMemory(int) {}diff --git a/internal/p2plimits/streams_test.go b/internal/p2plimits/streams_test.gonew file mode 100644index 00000000..3514d3c4--- /dev/null+++ b/internal/p2plimits/streams_test.go@@ -0,0 +1,93 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package p2plimits++import (+ "fmt"+ "testing"+ "time"++ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/protocol"+ "github.com/libp2p/go-libp2p/core/test"+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"+ "github.com/prometheus/client_golang/prometheus"+)++func newStreamsRefusedCounter() *prometheus.CounterVec {+ return prometheus.NewCounterVec(prometheus.CounterOpts{Name: "test_streams_refused_total"}, []string{"scope", "protocol"})+}++// A stream the manager refuses at a protocol scope is counted under that+// protocol, which libp2p's own gauge does not say; a granted one is not.+func TestStreamRefusalsCountTheProtocol(t *testing.T) {+ const proto protocol.ID = "/test/kad/1.0.0"+ limits := rcmgr.PartialLimitConfig{+ Protocol: map[protocol.ID]rcmgr.ResourceLimits{proto: Streams(1)},+ }.Build(rcmgr.InfiniteLimits)+ refused := newStreamsRefusedCounter()+ m, err := NewResourceManager(limits, 8, rcmgr.WithMetrics(CountStreamRefusals(refused, nil)))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = m.Close() }()++ p := test.RandPeerIDFatal(t)+ var scopes []network.StreamManagementScope+ for i := 0; i < 2; i++ {+ s, err := m.OpenStream(p, network.DirInbound)+ if err != nil {+ t.Fatalf("stream %d refused before any protocol: %v", i+1, err)+ }+ scopes = append(scopes, s)+ }+ if err := scopes[0].SetProtocol(proto); err != nil {+ t.Fatalf("first stream refused on %s: %v", proto, err)+ }+ if err := scopes[1].SetProtocol(proto); err == nil {+ t.Fatalf("second stream admitted on %s with a budget of one", proto)+ }+ if got := counterValue(t, refused.WithLabelValues(StreamScopeProtocol, string(proto))); got != 1 {+ t.Errorf("refusals on %s = %v, want 1", proto, got)+ }+ for _, s := range scopes {+ s.Done()+ }+}++func TestStreamRefusalsLogOneProtocolOncePerInterval(t *testing.T) {+ now := time.Unix(1_700_000_000, 0)+ var lines []string+ s := CountStreamRefusals(newStreamsRefusedCounter(), func(format string, args ...any) {+ lines = append(lines, fmt.Sprintf(format, args...))+ })+ s.SetClock(func() time.Time { return now })++ for i := 0; i < 3; i++ {+ s.BlockProtocol("/a")+ }+ s.BlockProtocolPeer("/a", "")+ s.BlockService("relay")+ if len(lines) != 3 {+ t.Fatalf("lines = %d, want one per scope and protocol: %q", len(lines), lines)+ }++ // The next line for a protocol carries what was refused since the last.+ now = now.Add(RefusalLogInterval)+ s.BlockProtocol("/a")+ if len(lines) != 4 || lines[3] != "[Stream] Refused 3 stream(s) on /a at the protocol scope since the last report; the peer sees its stream reset" {+ t.Errorf("after the interval: %q", lines[len(lines)-1])+ }+}diff --git a/internal/router/resource_limits.go b/internal/router/resource_limits.goindex ac029431..ffe76a30 100644--- a/internal/router/resource_limits.go+++ b/internal/router/resource_limits.go@@ -15,9 +15,12 @@package routerimport (+ "slices"+"github.com/google/agentmesh/api""github.com/google/agentmesh/internal/p2plimits""github.com/libp2p/go-libp2p"+ pubsub "github.com/libp2p/go-libp2p-pubsub""github.com/libp2p/go-libp2p/core/protocol"rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"@@ -25,6 +28,20 @@ import ("github.com/libp2p/go-libp2p/p2p/protocol/identify")+// memberProtocols are the protocols on which every member holds a stream+// open to its router for as long as it is connected (the DHT keeps one+// request stream per peer, gossip one per direction) or opens one when it+// joins. libp2p's default for a protocol it was not told about is a few+// hundred inbound streams in all, scaled from host memory: on the testnet it+// was 768, and the 769th member's DHT and gossip streams were refused, which+// kept the router out of that member's routing table and the member out of+// discovery (libp2p_rcmgr_blocked_resources{scope="protocol"} counted them,+// without saying which protocol).+var memberProtocols = slices.Concat(+ []protocol.ID{identify.ID, identify.IDPush, api.AuthProtocolID, api.DHTProtocolID},+ pubsub.GossipSubDefaultProtocols,+)+// connCapHeadroom is the resource manager's connection cap as a multiple of// the high watermark.const connCapHeadroom = 2@@ -73,6 +90,14 @@ func hubLimits(highWaterMark, conns, circuits int) rcmgr.ConcreteLimitConfig {relayStreams.Memory = rcmgr.LimitVal64(int64(sysBase) * 64 << 10)perPeerCircuits := streams(circuits)+ protocols := map[protocol.ID]rcmgr.ResourceLimits{+ circuit.ProtoIDv2Hop: relayStreams,+ circuit.ProtoIDv2Stop: relayStreams,+ }+ for _, p := range memberProtocols {+ protocols[p] = streams(sysBase)+ }+overrides := rcmgr.PartialLimitConfig{System: system,Transient: transient,@@ -83,13 +108,7 @@ func hubLimits(highWaterMark, conns, circuits int) rcmgr.ConcreteLimitConfig {ServicePeer: map[string]rcmgr.ResourceLimits{relay.ServiceName: perPeerCircuits,},- Protocol: map[protocol.ID]rcmgr.ResourceLimits{- identify.ID: streams(sysBase),- identify.IDPush: streams(sysBase),- api.AuthProtocolID: streams(sysBase),- circuit.ProtoIDv2Hop: relayStreams,- circuit.ProtoIDv2Stop: relayStreams,- },+ Protocol: protocols,ProtocolPeer: map[protocol.ID]rcmgr.ResourceLimits{circuit.ProtoIDv2Hop: perPeerCircuits,circuit.ProtoIDv2Stop: perPeerCircuits,diff --git a/internal/router/resource_limits_test.go b/internal/router/resource_limits_test.goindex e782201b..33d29c13 100644--- a/internal/router/resource_limits_test.go+++ b/internal/router/resource_limits_test.go@@ -15,10 +15,14 @@package routerimport (+ "slices""testing""github.com/google/agentmesh/api"+ dht "github.com/libp2p/go-libp2p-kad-dht"+ pubsub "github.com/libp2p/go-libp2p-pubsub"rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"+ mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto""github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay""github.com/libp2p/go-libp2p/p2p/protocol/identify"@@ -47,6 +51,13 @@ func TestHubLimitsFollowTheConnectionBudget(t *testing.T) {want("identify service inbound streams", cfg.Service[identify.ServiceName].StreamsInbound, highWaterMark)want("identify protocol inbound streams", cfg.Protocol[identify.ID].StreamsInbound, highWaterMark)want("auth protocol inbound streams", cfg.Protocol[api.AuthProtocolID].StreamsInbound, highWaterMark)+ // Every member holds one DHT stream and one gossip stream to its router+ // for as long as it is connected; libp2p's default capped them at a few+ // hundred in all and the rest of the fleet fell out of discovery.+ want("dht protocol inbound streams", cfg.Protocol[api.DHTProtocolID].StreamsInbound, highWaterMark)+ for _, p := range pubsub.GossipSubDefaultProtocols {+ want("gossip protocol "+string(p)+" inbound streams", cfg.Protocol[p].StreamsInbound, highWaterMark)+ }want("relay service inbound streams", cfg.Service[relay.ServiceName].StreamsInbound, highWaterMark)want("hop protocol inbound streams", cfg.Protocol[circuit.ProtoIDv2Hop].StreamsInbound, highWaterMark)// A destination many members call holds one stop stream per caller.@@ -60,3 +71,24 @@ func TestHubLimitsFollowTheConnectionBudget(t *testing.T) {small := hubLimits(100, 8, circuits).ToPartialLimitConfig()want("system inbound conns with a small watermark and cap", small.System.ConnsInbound, connCapHeadroom*DefaultHighWaterMark)}++// The DHT protocol the limits budget for is the one a DHT built the way the+// router builds it registers; a renamed prefix would silently fall back to+// libp2p's default budget.+func TestDHTProtocolIDIsWhatTheRouterServes(t *testing.T) {+ h, err := mocknet.New().GenPeer()+ if err != nil {+ t.Fatal(err)+ }+ kad, err := dht.New(h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(api.DHTProtocolPrefix))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = kad.Close() }()+ if got := h.Mux().Protocols(); !slices.Contains(got, api.DHTProtocolID) {+ t.Errorf("DHT server registers %v, not %s", got, api.DHTProtocolID)+ }+ if !slices.Contains(memberProtocols, api.DHTProtocolID) {+ t.Error("the DHT protocol is not among the member protocols the limits budget for")+ }+}diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.goindex 1c5e9bdb..6242f523 100644--- a/internal/router/resource_observer.go+++ b/internal/router/resource_observer.go@@ -33,10 +33,21 @@ var inboundRefusedTotal = promauto.NewCounterVec([]string{"reason"},)+var streamsRefusedTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "agentmesh_router_streams_refused_total",+ Help: "Streams the resource manager refused, by scope and protocol (or service); a member whose DHT or gossip stream is refused drops out of discovery",+ },+ []string{"scope", "protocol"},+)+func init() {for _, v := range p2plimits.Reasons {inboundRefusedTotal.WithLabelValues(v)}+ for _, p := range memberProtocols {+ streamsRefusedTotal.WithLabelValues(p2plimits.StreamScopeProtocol, string(p))+ }}// observeResourceManager counts the wrapped manager's refusals in@@ -45,6 +56,12 @@ func observeResourceManager(inner network.ResourceManager) *p2plimits.Observed {return p2plimits.Observe(inner, inboundRefusedTotal, logger.Warnf)}+// countStreamRefusals is the reporter the resource manager is built with;+// it counts refused streams in streamsRefusedTotal by protocol.+func countStreamRefusals() *p2plimits.StreamRefusals {+ return p2plimits.CountStreamRefusals(streamsRefusedTotal, logger.Warnf)+}+// connsPerSourceIPDesc exports the cap in force, so a dashboard can read the// refusal counter against it.var connsPerSourceIPDesc = prometheus.NewDesc(diff --git a/internal/router/router.go b/internal/router/router.goindex f776d45a..a4706d60 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -48,6 +48,7 @@ import ("github.com/libp2p/go-libp2p/core/host""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager""github.com/libp2p/go-libp2p/p2p/net/connmgr""github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay""github.com/libp2p/go-libp2p/p2p/security/noise"@@ -435,7 +436,7 @@ func (r *Router) Start() (retErr error) {// Every refusal the resource manager makes is counted and logged with// its source; a refused dialer only sees its connection close before TLS.- mgr, err := p2plimits.NewResourceManager(hubLimits(r.config.HighWaterMark, r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits), r.config.ConnsPerSourceIP)+ mgr, err := p2plimits.NewResourceManager(hubLimits(r.config.HighWaterMark, r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits), r.config.ConnsPerSourceIP, rcmgr.WithMetrics(countStreamRefusals()))if err != nil {return fmt.Errorf("failed to create resource manager: %w", err)}@@ -450,7 +451,7 @@ func (r *Router) Start() (retErr error) {// Setup DHTdhtOpts := []dht.Option{dht.Mode(dht.ModeServer),- dht.ProtocolPrefix("/mesh"),+ dht.ProtocolPrefix(api.DHTProtocolPrefix),}providerTTL := r.config.DHTProviderAddrTTLif providerTTL <= 0 {diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.mdindex 3c31ecfe..3135c7c6 100644--- a/site/content/docs/guides/scalability.md+++ b/site/content/docs/guides/scalability.md@@ -104,6 +104,16 @@ rising has more connections than two watermarks, which a rollout of amesh at capacity produces when the stopped router's members land on theothers; the fix is a router or a higher watermark.+The stream budgets follow the watermark per protocol, because a member+holds a stream open to its router for as long as it is connected on each+of the DHT, GossipSub, and its relay reservation, and libp2p's default+for a protocol it was not told about is a few hundred streams in all,+scaled from host memory. A router that refuses a member's DHT stream+drops out of that member's routing table, and the member out of+discovery, with no error at the member; the router counts the refusal in+`agentmesh_router_streams_refused_total{protocol}` and logs it once a+minute per protocol.+Routers connect to one another, every pair, to run the DHT and GossipSubacross the mesh; a router never sheds those connections. That is N × (N − 1)/ 2 connections for N routers, which is nothing at ten routers and a@@ -229,6 +239,7 @@ The router and the control plane serve Prometheus metrics on| `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. || `libp2p_relaysvc_connection_rejections_total{reason="resource limit exceeded"}` | A service has more callers than `--relay-max-circuits`. || `libp2p_rcmgr_blocked_resources` | libp2p refused a connection or stream on a scope. On a router this should stay flat. |+| `agentmesh_router_streams_refused_total{scope,protocol}`, `agentmesh_node_streams_refused_total{scope,protocol}` | The streams behind that gauge, by protocol. `/mesh/kad/1.0.0` or `/meshsub/*` at the `protocol` scope on a router is members past the stream budget of a protocol, which keeps them out of discovery; `agentmesh_router_dht_routing_table_size` falls with it. || `agentmesh_router_auth_handshakes_total{result="read_failed"}` | Members that timed out mid-handshake: the router is slow, usually CPU. || `agentmesh_router_draining` | The router is stopping. Exclude it from capacity sums. || `agentmesh_control_plane_http_requests_total{code="429"}` | Members turned away by the enrollment limiter; they retry. |diff --git a/site/content/docs/reference/node-api.md b/site/content/docs/reference/node-api.mdindex d7b9a6b2..1dbb2a5d 100644--- a/site/content/docs/reference/node-api.md+++ b/site/content/docs/reference/node-api.md@@ -66,9 +66,11 @@ Authorization Server.listener without authentication, for scrapers that hold no token. Besidesthe gauges `/debug/mesh-info` answers on demand, `/metrics` carries`agentmesh_node_inbound_connections_refused_total{reason}`, the inbound-connections the node turned away before the handshake, and+connections the node turned away before the handshake,`agentmesh_node_conns_per_source_ip_limit`, the per-address cap they are-read against; see the scalability guide.+read against, and `agentmesh_node_streams_refused_total{scope,protocol}`,+the streams it refused on an admitted connection; see the scalability+guide.## Token endpoint (`/oauth/token`) and revocation (`/oauth/revoke`)diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex a7cb5fdc..fd4f159a 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -120,6 +120,7 @@ The metrics address exposes, next to the Go runtime metrics:| `agentmesh_router_goaway_sent_total{reason,result}` | Members sent away, by reason (`DRAINING`, `OVERLOADED`) and outcome: `ok` when the member took the message, `unsupported` when it does not speak the protocol and was closed instead, `failed` when the stream broke. A rising `OVERLOADED` count on a router that is not draining is a fleet that has outgrown its routers. || `agentmesh_router_unauthenticated_connections_closed_total` | Connections closed at `--auth-deadline` without a handshake. A steady rate is DHT traffic from members attached to other routers; a burst is a scanner. || `agentmesh_router_inbound_connections_refused_total` | Inbound connections refused by the per-address cap. |+| `agentmesh_router_streams_refused_total{scope,protocol}` | Streams the resource manager refused, by scope and protocol or service. A count on `/mesh/kad/1.0.0` or a `/meshsub/*` protocol is members the router keeps out of discovery; see the scalability guide. || `libp2p_relaysvc_*` | The relay service's own counters: `reservations_total{type}` and `connections_total{type}` (`opened`, `closed`, `renewed`), requests by response status, rejections and bytes relayed. The difference between `opened` and `closed` is the number in use; read it against `agentmesh_router_relay_limit` to see how close the relay is to its budget. |A fleet is at capacity when `agentmesh_router_connections{direction="inbound"}`