Skip to content

Agent Mesh

router, node: budget the DHT and gossip streams per member, count refused streams by protocol

Antonio Ojea9c453b903e37
diff --git a/api/network.go b/api/network.go
index be60a092..02a5a458 100644
--- a/api/network.go
+++ b/api/network.go
@@ -44,6 +44,12 @@ const (
// it will no longer hold it; the stream carries one RouterGoAway.
GoAwayProtocolID protocol.ID = "/mesh/goaway/1.0.0"
+ // DHTProtocolPrefix namespaces the mesh's Kademlia DHT away from the
+ // public IPFS one; the protocol every member and router registers for
+ // it is DHTProtocolID.
+ DHTProtocolPrefix protocol.ID = "/mesh"
+ DHTProtocolID protocol.ID = DHTProtocolPrefix + "/kad/1.0.0"
+
// GossipEvents is the GossipSub topic used to broadcast mesh event updates (e.g., node bans).
GossipEvents = "/mesh/events/v1"
diff --git a/internal/node/node.go b/internal/node/node.go
index 3caf6228..5e6eb174 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -689,7 +689,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
// Initialize Rendezvous (DHT Client)
dhtOpts := []dht.Option{
dht.Mode(dht.ModeAuto),
- dht.ProtocolPrefix("/mesh"),
+ dht.ProtocolPrefix(api.DHTProtocolPrefix),
}
providerTTL := n.config.DHTProviderAddrTTL
if providerTTL <= 0 {
diff --git a/internal/node/resource_limits.go b/internal/node/resource_limits.go
index 7c6fff47..a0d7ef21 100644
--- a/internal/node/resource_limits.go
+++ b/internal/node/resource_limits.go
@@ -15,9 +15,12 @@
package node
import (
+ "slices"
+
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/p2plimits"
"github.com/libp2p/go-libp2p"
+ pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/protocol"
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
@@ -91,19 +94,36 @@ func nodeLimits(conns int) rcmgr.ConcreteLimitConfig {
Service: map[string]rcmgr.ResourceLimits{
identify.ServiceName: p2plimits.Streams(conns),
},
- Protocol: map[protocol.ID]rcmgr.ResourceLimits{
- identify.ID: p2plimits.Streams(conns),
- identify.IDPush: p2plimits.Streams(conns),
- api.AuthProtocolID: p2plimits.Streams(conns),
- },
+ Protocol: map[protocol.ID]rcmgr.ResourceLimits{},
}
+ // One stream per caller for as long as it is connected (its DHT request
+ // stream, gossip), or one when it arrives; libp2p's default for a
+ // protocol it was not told about scales from host memory. A caller holds
+ // one MCP stream per session, so that protocol gets the per-connection
+ // share of the system budget.
+ for _, p := range slices.Concat(
+ []protocol.ID{identify.ID, identify.IDPush, api.AuthProtocolID, api.DHTProtocolID},
+ pubsub.GossipSubDefaultProtocols,
+ ) {
+ overrides.Protocol[p] = p2plimits.Streams(conns)
+ }
+ overrides.Protocol[api.MCPProtocolID] = p2plimits.Streams(streamsPerConn * conns)
return overrides.Build(scaled)
}
+var streamsRefusedTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_node_streams_refused_total",
+ Help: "Streams the resource manager refused, by scope and protocol (or service)",
+ },
+ []string{"scope", "protocol"},
+)
+
// newResourceManager is the resource manager the node's host runs with:
// nodeLimits, the per-source cap, and every refusal counted and logged.
func newResourceManager() (network.ResourceManager, error) {
- mgr, err := p2plimits.NewResourceManager(nodeLimits(ConnHighWaterMark), ConnsPerSourceIP)
+ mgr, err := p2plimits.NewResourceManager(nodeLimits(ConnHighWaterMark), ConnsPerSourceIP,
+ rcmgr.WithMetrics(p2plimits.CountStreamRefusals(streamsRefusedTotal, logger.Warnf)))
if err != nil {
return nil, err
}
diff --git a/internal/node/resource_limits_test.go b/internal/node/resource_limits_test.go
index a43ebc98..5542233b 100644
--- a/internal/node/resource_limits_test.go
+++ b/internal/node/resource_limits_test.go
@@ -18,6 +18,7 @@ import (
"testing"
"github.com/google/agentmesh/api"
+ pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/connmgr"
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
"github.com/libp2p/go-libp2p/p2p/protocol/identify"
@@ -39,6 +40,12 @@ func TestNodeLimitsFollowTheConnectionBudget(t *testing.T) {
want("transient inbound conns", cfg.Transient.ConnsInbound, ConnHighWaterMark)
want("identify service inbound streams", cfg.Service[identify.ServiceName].StreamsInbound, ConnHighWaterMark)
want("auth protocol inbound streams", cfg.Protocol[api.AuthProtocolID].StreamsInbound, ConnHighWaterMark)
+ // A caller holds its MCP, DHT and gossip streams for as long as it stays.
+ want("mcp protocol inbound streams", cfg.Protocol[api.MCPProtocolID].StreamsInbound, 8*ConnHighWaterMark)
+ want("dht protocol inbound streams", cfg.Protocol[api.DHTProtocolID].StreamsInbound, ConnHighWaterMark)
+ for _, p := range pubsub.GossipSubDefaultProtocols {
+ want("gossip protocol "+string(p)+" inbound streams", cfg.Protocol[p].StreamsInbound, ConnHighWaterMark)
+ }
if ConnsPerSourceIP != ConnHighWaterMark/4 {
t.Errorf("ConnsPerSourceIP = %d, want a quarter of the high watermark, %d", ConnsPerSourceIP, ConnHighWaterMark/4)
diff --git a/internal/p2plimits/limits.go b/internal/p2plimits/limits.go
index dccdb0ac..98fccf06 100644
--- a/internal/p2plimits/limits.go
+++ b/internal/p2plimits/limits.go
@@ -71,8 +71,8 @@ const libp2pDefaultConnsPerIP = 8
// connection rate limit scaled to it. Loopback is exempt from both, as in
// libp2p's default, so a local mesh never meets them. A cap under one
// would refuse every peer, so it is an error here rather than a silent
-// listener.
-func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int) (network.ResourceManager, error) {
+// listener. opts are passed on, for a MetricsReporter among others.
+func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int, opts ...rcmgr.Option) (network.ResourceManager, error) {
if perIP <= 0 {
return nil, fmt.Errorf("connections per source address must be at least 1, got %d", perIP)
}
@@ -94,11 +94,13 @@ func NewResourceManager(limits rcmgr.ConcreteLimitConfig, perIP int) (network.Re
}
return rcmgr.NewResourceManager(
rcmgr.NewFixedLimiter(limits),
- rcmgr.WithLimitPerSubnet(
- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: perIP}},
- []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: perIP}},
- ),
- rcmgr.WithConnRateLimiters(connRateLimiter),
+ append([]rcmgr.Option{
+ rcmgr.WithLimitPerSubnet(
+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: perIP}},
+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: perIP}},
+ ),
+ rcmgr.WithConnRateLimiters(connRateLimiter),
+ }, opts...)...,
)
}
diff --git a/internal/p2plimits/streams.go b/internal/p2plimits/streams.go
new file mode 100644
index 00000000..7e7e124e
--- /dev/null
+++ b/internal/p2plimits/streams.go
@@ -0,0 +1,123 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package p2plimits
+
+import (
+ "sync"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/protocol"
+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+// The scopes a stream refusal is counted under. libp2p's own gauge
+// (libp2p_rcmgr_blocked_resources) carries the scope but not the protocol,
+// and a hub needs the protocol to tell a flood from a limit set too low.
+const (
+ StreamScopeProtocol = "protocol"
+ StreamScopeProtocolPeer = "protocol_peer"
+ StreamScopeService = "service"
+ StreamScopeServicePeer = "service_peer"
+ StreamScopePeer = "peer"
+ // StreamScopeConn is the system or transient scope, where the manager
+ // does not yet know the protocol.
+ StreamScopeConn = "conn"
+)
+
+// StreamRefusals counts the streams the resource manager refused, by scope
+// and protocol (or service), and logs each scope and name once per
+// RefusalLogInterval. It is libp2p's MetricsReporter; the Allow side is a
+// no-op.
+type StreamRefusals struct {
+ refused *prometheus.CounterVec
+ warnf func(format string, args ...any)
+ mu sync.Mutex
+ since map[string]uint64
+ lastLogAt map[string]time.Time
+ now func() time.Time
+}
+
+var _ rcmgr.MetricsReporter = (*StreamRefusals)(nil)
+
+// CountStreamRefusals returns a reporter for rcmgr.WithMetrics. refused
+// carries the labels scope and protocol; a nil warnf only counts.
+func CountStreamRefusals(refused *prometheus.CounterVec, warnf func(format string, args ...any)) *StreamRefusals {
+ if warnf == nil {
+ warnf = func(string, ...any) {}
+ }
+ return &StreamRefusals{
+ refused: refused,
+ warnf: warnf,
+ since: map[string]uint64{},
+ lastLogAt: map[string]time.Time{},
+ now: time.Now,
+ }
+}
+
+// SetClock replaces the clock the log interval is measured on.
+func (s *StreamRefusals) SetClock(now func() time.Time) { s.now = now }
+
+func (s *StreamRefusals) refusedStream(scope, name string) {
+ s.refused.WithLabelValues(scope, name).Inc()
+ key := scope + " " + name
+ s.mu.Lock()
+ s.since[key]++
+ n := s.since[key]
+ now := s.now()
+ last, seen := s.lastLogAt[key]
+ shouldLog := !seen || now.Sub(last) >= RefusalLogInterval
+ if shouldLog {
+ s.lastLogAt[key] = now
+ s.since[key] = 0
+ }
+ s.mu.Unlock()
+ if shouldLog {
+ s.warnf("[Stream] Refused %d stream(s) on %s at the %s scope since the last report; the peer sees its stream reset", n, name, scope)
+ }
+}
+
+func (s *StreamRefusals) BlockStream(_ peer.ID, _ network.Direction) {
+ s.refusedStream(StreamScopeConn, "")
+}
+
+func (s *StreamRefusals) BlockPeer(peer.ID) { s.refusedStream(StreamScopePeer, "") }
+
+func (s *StreamRefusals) BlockProtocol(proto protocol.ID) {
+ s.refusedStream(StreamScopeProtocol, string(proto))
+}
+
+func (s *StreamRefusals) BlockProtocolPeer(proto protocol.ID, _ peer.ID) {
+ s.refusedStream(StreamScopeProtocolPeer, string(proto))
+}
+
+func (s *StreamRefusals) BlockService(svc string) { s.refusedStream(StreamScopeService, svc) }
+
+func (s *StreamRefusals) BlockServicePeer(svc string, _ peer.ID) {
+ s.refusedStream(StreamScopeServicePeer, svc)
+}
+
+// Connections and memory are observed elsewhere (Observed and libp2p's own
+// gauges); allowances are not of interest.
+func (s *StreamRefusals) AllowConn(network.Direction, bool) {}
+func (s *StreamRefusals) BlockConn(network.Direction, bool) {}
+func (s *StreamRefusals) AllowStream(peer.ID, network.Direction) {}
+func (s *StreamRefusals) AllowPeer(peer.ID) {}
+func (s *StreamRefusals) AllowProtocol(protocol.ID) {}
+func (s *StreamRefusals) AllowService(string) {}
+func (s *StreamRefusals) AllowMemory(int) {}
+func (s *StreamRefusals) BlockMemory(int) {}
diff --git a/internal/p2plimits/streams_test.go b/internal/p2plimits/streams_test.go
new file mode 100644
index 00000000..3514d3c4
--- /dev/null
+++ b/internal/p2plimits/streams_test.go
@@ -0,0 +1,93 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package p2plimits
+
+import (
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/protocol"
+ "github.com/libp2p/go-libp2p/core/test"
+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+func newStreamsRefusedCounter() *prometheus.CounterVec {
+ return prometheus.NewCounterVec(prometheus.CounterOpts{Name: "test_streams_refused_total"}, []string{"scope", "protocol"})
+}
+
+// A stream the manager refuses at a protocol scope is counted under that
+// protocol, which libp2p's own gauge does not say; a granted one is not.
+func TestStreamRefusalsCountTheProtocol(t *testing.T) {
+ const proto protocol.ID = "/test/kad/1.0.0"
+ limits := rcmgr.PartialLimitConfig{
+ Protocol: map[protocol.ID]rcmgr.ResourceLimits{proto: Streams(1)},
+ }.Build(rcmgr.InfiniteLimits)
+ refused := newStreamsRefusedCounter()
+ m, err := NewResourceManager(limits, 8, rcmgr.WithMetrics(CountStreamRefusals(refused, nil)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = m.Close() }()
+
+ p := test.RandPeerIDFatal(t)
+ var scopes []network.StreamManagementScope
+ for i := 0; i < 2; i++ {
+ s, err := m.OpenStream(p, network.DirInbound)
+ if err != nil {
+ t.Fatalf("stream %d refused before any protocol: %v", i+1, err)
+ }
+ scopes = append(scopes, s)
+ }
+ if err := scopes[0].SetProtocol(proto); err != nil {
+ t.Fatalf("first stream refused on %s: %v", proto, err)
+ }
+ if err := scopes[1].SetProtocol(proto); err == nil {
+ t.Fatalf("second stream admitted on %s with a budget of one", proto)
+ }
+ if got := counterValue(t, refused.WithLabelValues(StreamScopeProtocol, string(proto))); got != 1 {
+ t.Errorf("refusals on %s = %v, want 1", proto, got)
+ }
+ for _, s := range scopes {
+ s.Done()
+ }
+}
+
+func TestStreamRefusalsLogOneProtocolOncePerInterval(t *testing.T) {
+ now := time.Unix(1_700_000_000, 0)
+ var lines []string
+ s := CountStreamRefusals(newStreamsRefusedCounter(), func(format string, args ...any) {
+ lines = append(lines, fmt.Sprintf(format, args...))
+ })
+ s.SetClock(func() time.Time { return now })
+
+ for i := 0; i < 3; i++ {
+ s.BlockProtocol("/a")
+ }
+ s.BlockProtocolPeer("/a", "")
+ s.BlockService("relay")
+ if len(lines) != 3 {
+ t.Fatalf("lines = %d, want one per scope and protocol: %q", len(lines), lines)
+ }
+
+ // The next line for a protocol carries what was refused since the last.
+ now = now.Add(RefusalLogInterval)
+ s.BlockProtocol("/a")
+ if len(lines) != 4 || lines[3] != "[Stream] Refused 3 stream(s) on /a at the protocol scope since the last report; the peer sees its stream reset" {
+ t.Errorf("after the interval: %q", lines[len(lines)-1])
+ }
+}
diff --git a/internal/router/resource_limits.go b/internal/router/resource_limits.go
index ac029431..ffe76a30 100644
--- a/internal/router/resource_limits.go
+++ b/internal/router/resource_limits.go
@@ -15,9 +15,12 @@
package router
import (
+ "slices"
+
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/p2plimits"
"github.com/libp2p/go-libp2p"
+ pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/protocol"
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
@@ -25,6 +28,20 @@ import (
"github.com/libp2p/go-libp2p/p2p/protocol/identify"
)
+// memberProtocols are the protocols on which every member holds a stream
+// open to its router for as long as it is connected (the DHT keeps one
+// request stream per peer, gossip one per direction) or opens one when it
+// joins. libp2p's default for a protocol it was not told about is a few
+// hundred inbound streams in all, scaled from host memory: on the testnet it
+// was 768, and the 769th member's DHT and gossip streams were refused, which
+// kept the router out of that member's routing table and the member out of
+// discovery (libp2p_rcmgr_blocked_resources{scope="protocol"} counted them,
+// without saying which protocol).
+var memberProtocols = slices.Concat(
+ []protocol.ID{identify.ID, identify.IDPush, api.AuthProtocolID, api.DHTProtocolID},
+ pubsub.GossipSubDefaultProtocols,
+)
+
// connCapHeadroom is the resource manager's connection cap as a multiple of
// the high watermark.
const connCapHeadroom = 2
@@ -73,6 +90,14 @@ func hubLimits(highWaterMark, conns, circuits int) rcmgr.ConcreteLimitConfig {
relayStreams.Memory = rcmgr.LimitVal64(int64(sysBase) * 64 << 10)
perPeerCircuits := streams(circuits)
+ protocols := map[protocol.ID]rcmgr.ResourceLimits{
+ circuit.ProtoIDv2Hop: relayStreams,
+ circuit.ProtoIDv2Stop: relayStreams,
+ }
+ for _, p := range memberProtocols {
+ protocols[p] = streams(sysBase)
+ }
+
overrides := rcmgr.PartialLimitConfig{
System: system,
Transient: transient,
@@ -83,13 +108,7 @@ func hubLimits(highWaterMark, conns, circuits int) rcmgr.ConcreteLimitConfig {
ServicePeer: map[string]rcmgr.ResourceLimits{
relay.ServiceName: perPeerCircuits,
},
- Protocol: map[protocol.ID]rcmgr.ResourceLimits{
- identify.ID: streams(sysBase),
- identify.IDPush: streams(sysBase),
- api.AuthProtocolID: streams(sysBase),
- circuit.ProtoIDv2Hop: relayStreams,
- circuit.ProtoIDv2Stop: relayStreams,
- },
+ Protocol: protocols,
ProtocolPeer: map[protocol.ID]rcmgr.ResourceLimits{
circuit.ProtoIDv2Hop: perPeerCircuits,
circuit.ProtoIDv2Stop: perPeerCircuits,
diff --git a/internal/router/resource_limits_test.go b/internal/router/resource_limits_test.go
index e782201b..33d29c13 100644
--- a/internal/router/resource_limits_test.go
+++ b/internal/router/resource_limits_test.go
@@ -15,10 +15,14 @@
package router
import (
+ "slices"
"testing"
"github.com/google/agentmesh/api"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ pubsub "github.com/libp2p/go-libp2p-pubsub"
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
+ mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
"github.com/libp2p/go-libp2p/p2p/protocol/identify"
@@ -47,6 +51,13 @@ func TestHubLimitsFollowTheConnectionBudget(t *testing.T) {
want("identify service inbound streams", cfg.Service[identify.ServiceName].StreamsInbound, highWaterMark)
want("identify protocol inbound streams", cfg.Protocol[identify.ID].StreamsInbound, highWaterMark)
want("auth protocol inbound streams", cfg.Protocol[api.AuthProtocolID].StreamsInbound, highWaterMark)
+ // Every member holds one DHT stream and one gossip stream to its router
+ // for as long as it is connected; libp2p's default capped them at a few
+ // hundred in all and the rest of the fleet fell out of discovery.
+ want("dht protocol inbound streams", cfg.Protocol[api.DHTProtocolID].StreamsInbound, highWaterMark)
+ for _, p := range pubsub.GossipSubDefaultProtocols {
+ want("gossip protocol "+string(p)+" inbound streams", cfg.Protocol[p].StreamsInbound, highWaterMark)
+ }
want("relay service inbound streams", cfg.Service[relay.ServiceName].StreamsInbound, highWaterMark)
want("hop protocol inbound streams", cfg.Protocol[circuit.ProtoIDv2Hop].StreamsInbound, highWaterMark)
// A destination many members call holds one stop stream per caller.
@@ -60,3 +71,24 @@ func TestHubLimitsFollowTheConnectionBudget(t *testing.T) {
small := hubLimits(100, 8, circuits).ToPartialLimitConfig()
want("system inbound conns with a small watermark and cap", small.System.ConnsInbound, connCapHeadroom*DefaultHighWaterMark)
}
+
+// The DHT protocol the limits budget for is the one a DHT built the way the
+// router builds it registers; a renamed prefix would silently fall back to
+// libp2p's default budget.
+func TestDHTProtocolIDIsWhatTheRouterServes(t *testing.T) {
+ h, err := mocknet.New().GenPeer()
+ if err != nil {
+ t.Fatal(err)
+ }
+ kad, err := dht.New(h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(api.DHTProtocolPrefix))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = kad.Close() }()
+ if got := h.Mux().Protocols(); !slices.Contains(got, api.DHTProtocolID) {
+ t.Errorf("DHT server registers %v, not %s", got, api.DHTProtocolID)
+ }
+ if !slices.Contains(memberProtocols, api.DHTProtocolID) {
+ t.Error("the DHT protocol is not among the member protocols the limits budget for")
+ }
+}
diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.go
index 1c5e9bdb..6242f523 100644
--- a/internal/router/resource_observer.go
+++ b/internal/router/resource_observer.go
@@ -33,10 +33,21 @@ var inboundRefusedTotal = promauto.NewCounterVec(
[]string{"reason"},
)
+var streamsRefusedTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_router_streams_refused_total",
+ Help: "Streams the resource manager refused, by scope and protocol (or service); a member whose DHT or gossip stream is refused drops out of discovery",
+ },
+ []string{"scope", "protocol"},
+)
+
func init() {
for _, v := range p2plimits.Reasons {
inboundRefusedTotal.WithLabelValues(v)
}
+ for _, p := range memberProtocols {
+ streamsRefusedTotal.WithLabelValues(p2plimits.StreamScopeProtocol, string(p))
+ }
}
// observeResourceManager counts the wrapped manager's refusals in
@@ -45,6 +56,12 @@ func observeResourceManager(inner network.ResourceManager) *p2plimits.Observed {
return p2plimits.Observe(inner, inboundRefusedTotal, logger.Warnf)
}
+// countStreamRefusals is the reporter the resource manager is built with;
+// it counts refused streams in streamsRefusedTotal by protocol.
+func countStreamRefusals() *p2plimits.StreamRefusals {
+ return p2plimits.CountStreamRefusals(streamsRefusedTotal, logger.Warnf)
+}
+
// connsPerSourceIPDesc exports the cap in force, so a dashboard can read the
// refusal counter against it.
var connsPerSourceIPDesc = prometheus.NewDesc(
diff --git a/internal/router/router.go b/internal/router/router.go
index f776d45a..a4706d60 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -48,6 +48,7 @@ import (
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
"github.com/libp2p/go-libp2p/p2p/security/noise"
@@ -435,7 +436,7 @@ func (r *Router) Start() (retErr error) {
// Every refusal the resource manager makes is counted and logged with
// its source; a refused dialer only sees its connection close before TLS.
- mgr, err := p2plimits.NewResourceManager(hubLimits(r.config.HighWaterMark, r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits), r.config.ConnsPerSourceIP)
+ mgr, err := p2plimits.NewResourceManager(hubLimits(r.config.HighWaterMark, r.config.ConnsPerSourceIP, r.config.RelayMaxCircuits), r.config.ConnsPerSourceIP, rcmgr.WithMetrics(countStreamRefusals()))
if err != nil {
return fmt.Errorf("failed to create resource manager: %w", err)
}
@@ -450,7 +451,7 @@ func (r *Router) Start() (retErr error) {
// Setup DHT
dhtOpts := []dht.Option{
dht.Mode(dht.ModeServer),
- dht.ProtocolPrefix("/mesh"),
+ dht.ProtocolPrefix(api.DHTProtocolPrefix),
}
providerTTL := r.config.DHTProviderAddrTTL
if providerTTL <= 0 {
diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.md
index 3c31ecfe..3135c7c6 100644
--- a/site/content/docs/guides/scalability.md
+++ b/site/content/docs/guides/scalability.md
@@ -104,6 +104,16 @@ rising has more connections than two watermarks, which a rollout of a
mesh at capacity produces when the stopped router's members land on the
others; the fix is a router or a higher watermark.
+The stream budgets follow the watermark per protocol, because a member
+holds a stream open to its router for as long as it is connected on each
+of the DHT, GossipSub, and its relay reservation, and libp2p's default
+for a protocol it was not told about is a few hundred streams in all,
+scaled from host memory. A router that refuses a member's DHT stream
+drops out of that member's routing table, and the member out of
+discovery, with no error at the member; the router counts the refusal in
+`agentmesh_router_streams_refused_total{protocol}` and logs it once a
+minute per protocol.
+
Routers connect to one another, every pair, to run the DHT and GossipSub
across the mesh; a router never sheds those connections. That is N × (N − 1)
/ 2 connections for N routers, which is nothing at ten routers and a
@@ -229,6 +239,7 @@ The router and the control plane serve Prometheus metrics on
| `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. |
| `libp2p_relaysvc_connection_rejections_total{reason="resource limit exceeded"}` | A service has more callers than `--relay-max-circuits`. |
| `libp2p_rcmgr_blocked_resources` | libp2p refused a connection or stream on a scope. On a router this should stay flat. |
+| `agentmesh_router_streams_refused_total{scope,protocol}`, `agentmesh_node_streams_refused_total{scope,protocol}` | The streams behind that gauge, by protocol. `/mesh/kad/1.0.0` or `/meshsub/*` at the `protocol` scope on a router is members past the stream budget of a protocol, which keeps them out of discovery; `agentmesh_router_dht_routing_table_size` falls with it. |
| `agentmesh_router_auth_handshakes_total{result="read_failed"}` | Members that timed out mid-handshake: the router is slow, usually CPU. |
| `agentmesh_router_draining` | The router is stopping. Exclude it from capacity sums. |
| `agentmesh_control_plane_http_requests_total{code="429"}` | Members turned away by the enrollment limiter; they retry. |
diff --git a/site/content/docs/reference/node-api.md b/site/content/docs/reference/node-api.md
index d7b9a6b2..1dbb2a5d 100644
--- a/site/content/docs/reference/node-api.md
+++ b/site/content/docs/reference/node-api.md
@@ -66,9 +66,11 @@ Authorization Server.
listener without authentication, for scrapers that hold no token. Besides
the gauges `/debug/mesh-info` answers on demand, `/metrics` carries
`agentmesh_node_inbound_connections_refused_total{reason}`, the inbound
-connections the node turned away before the handshake, and
+connections the node turned away before the handshake,
`agentmesh_node_conns_per_source_ip_limit`, the per-address cap they are
-read against; see the scalability guide.
+read against, and `agentmesh_node_streams_refused_total{scope,protocol}`,
+the streams it refused on an admitted connection; see the scalability
+guide.
## Token endpoint (`/oauth/token`) and revocation (`/oauth/revoke`)
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index a7cb5fdc..fd4f159a 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -120,6 +120,7 @@ The metrics address exposes, next to the Go runtime metrics:
| `agentmesh_router_goaway_sent_total{reason,result}` | Members sent away, by reason (`DRAINING`, `OVERLOADED`) and outcome: `ok` when the member took the message, `unsupported` when it does not speak the protocol and was closed instead, `failed` when the stream broke. A rising `OVERLOADED` count on a router that is not draining is a fleet that has outgrown its routers. |
| `agentmesh_router_unauthenticated_connections_closed_total` | Connections closed at `--auth-deadline` without a handshake. A steady rate is DHT traffic from members attached to other routers; a burst is a scanner. |
| `agentmesh_router_inbound_connections_refused_total` | Inbound connections refused by the per-address cap. |
+| `agentmesh_router_streams_refused_total{scope,protocol}` | Streams the resource manager refused, by scope and protocol or service. A count on `/mesh/kad/1.0.0` or a `/meshsub/*` protocol is members the router keeps out of discovery; see the scalability guide. |
| `libp2p_relaysvc_*` | The relay service's own counters: `reservations_total{type}` and `connections_total{type}` (`opened`, `closed`, `renewed`), requests by response status, rejections and bytes relayed. The difference between `opened` and `closed` is the number in use; read it against `agentmesh_router_relay_limit` to see how close the relay is to its budget. |
A fleet is at capacity when `agentmesh_router_connections{direction="inbound"}`