Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node, router: name the connection at QUIC's stream limit, count how a caller fares reaching a provider
Antonio Ojeae46a2c038713
diff --git a/internal/node/labels_gate.go b/internal/node/labels_gate.goindex 0f5164d3..293e9bf2 100644--- a/internal/node/labels_gate.go+++ b/internal/node/labels_gate.go@@ -28,6 +28,7 @@ import ("github.com/google/agentmesh/internal/identity""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-libp2p/p2p/net/swarm""github.com/libp2p/go-msgio""google.golang.org/protobuf/proto")@@ -98,6 +99,48 @@ const (// is the mesh's state and a caller may retry it; the second is a verdict.var ErrProviderUnreachable = errors.New("provider unreachable")+// How reaching a provider for its credential went; the label values of+// providerReachTotal. The caller sees one 502 for all of the failures, so+// the counter tells them apart: the relay refusing the circuit, the dial+// timing out (a relay's connection to the provider at its stream limit,+// or a provider gone), no address to dial, the provider closing the+// stream before answering.+const (+ reachOK = "ok"+ reachRelayRefused = "relay_refused"+ reachDialTimeout = "dial_timeout"+ reachNoAddress = "no_address"+ reachDialFailed = "dial_failed"+ reachNoAnswer = "no_answer"+ reachAuthRejected = "auth_rejected"+ reachOtherFailure = "other"+ relayRefusedMarker = "error opening relay circuit"+)++// reachFailure classifies why a provider was not reached.+func reachFailure(err error) string {+ var dialErr *swarm.DialError+ switch {+ case errors.As(err, &dialErr):+ switch {+ case dialErr.Timeout(), errors.Is(err, context.DeadlineExceeded):+ return reachDialTimeout+ case errors.Is(err, swarm.ErrNoAddresses), errors.Is(err, swarm.ErrNoGoodAddresses):+ return reachNoAddress+ case strings.Contains(err.Error(), relayRefusedMarker):+ return reachRelayRefused+ }+ return reachDialFailed+ case errors.Is(err, context.DeadlineExceeded):+ return reachDialTimeout+ case strings.Contains(err.Error(), "read auth response"):+ return reachNoAnswer+ case strings.Contains(err.Error(), "auth rejected"):+ return reachAuthRejected+ }+ return reachOtherFailure+}+// labelGateStatus is the HTTP answer for a label-gate failure, in the// order http.Error takes it: 502 when the provider was never heard from, so// a caller does not read a dead or busy destination as a denial of its@@ -174,8 +217,10 @@ func (n *AgentMeshNode) VerifyPeerLabels(ctx context.Context, peerID peer.ID, reproviderBiscuit, err := n.fetchPeerBiscuit(ctx, peerID)if err != nil {+ providerReachTotal.WithLabelValues(reachFailure(err)).Inc()return fmt.Errorf("provider %s labels unverifiable: %w: %w", peerID, ErrProviderUnreachable, err)}+ providerReachTotal.WithLabelValues(reachOK).Inc()if err := n.checkPeerLabels(providerBiscuit, peerID, required); err != nil {return err}diff --git a/internal/node/labels_gate_test.go b/internal/node/labels_gate_test.goindex d2f30720..5ef50921 100644--- a/internal/node/labels_gate_test.go+++ b/internal/node/labels_gate_test.go@@ -18,6 +18,7 @@ import ("context""crypto/ed25519""errors"+ "fmt""net/http""testing""time"@@ -27,6 +28,8 @@ import (lru "github.com/hashicorp/golang-lru/v2""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-libp2p/p2p/net/swarm"+ "github.com/multiformats/go-multiaddr")// The gate guards calls that ride relayed connections (egress proxy, MCP@@ -329,3 +332,30 @@ func TestLabelGateKeyIsUnambiguousWithSeparatorsInValues(t *testing.T) {}}}++// A provider the caller did not reach is one 502 whatever happened; the+// counter tells the cases apart from the error libp2p hands back.+func TestReachFailureClassifiesWhatKeptTheProviderAway(t *testing.T) {+ peerID := peer.ID("12D3KooWtest")+ relayRefused := &swarm.DialError{Peer: peerID, Cause: swarm.ErrAllDialsFailed, DialErrors: []swarm.TransportError{+ {Address: multiaddr.StringCast("/p2p-circuit"), Cause: errors.New("error opening relay circuit: PERMISSION_DENIED (202)")},+ }}+ timedOut := &swarm.DialError{Peer: peerID, Cause: context.DeadlineExceeded}+ for _, tc := range []struct {+ err error+ want string+ }{+ {fmt.Errorf("failed to open auth stream: %w", relayRefused), reachRelayRefused},+ {fmt.Errorf("failed to open auth stream: %w", timedOut), reachDialTimeout},+ {fmt.Errorf("failed to open auth stream: %w", &swarm.DialError{Peer: peerID, Cause: swarm.ErrNoAddresses}), reachNoAddress},+ {fmt.Errorf("failed to open auth stream: %w", &swarm.DialError{Peer: peerID, Cause: swarm.ErrAllDialsFailed, DialErrors: []swarm.TransportError{{Cause: errors.New("connection refused")}}}), reachDialFailed},+ {fmt.Errorf("failed to open auth stream: %w", context.DeadlineExceeded), reachDialTimeout},+ {errors.New("read auth response (peer may predate mutual auth): EOF"), reachNoAnswer},+ {errors.New("auth rejected: expired"), reachAuthRejected},+ {errors.New("something else"), reachOtherFailure},+ } {+ if got := reachFailure(tc.err); got != tc.want {+ t.Errorf("reachFailure(%v) = %s, want %s", tc.err, got, tc.want)+ }+ }+}diff --git a/internal/node/node.go b/internal/node/node.goindex 5e6eb174..946f01cd 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -42,6 +42,7 @@ import (cpclient "github.com/google/agentmesh/internal/controlplane/client""github.com/google/agentmesh/internal/identity"samdiscovery "github.com/google/agentmesh/internal/node/discovery"+ "github.com/google/agentmesh/internal/p2plimits""github.com/google/agentmesh/internal/ratelimit"lru "github.com/hashicorp/golang-lru/v2""github.com/ipfs/go-cid"@@ -799,6 +800,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {// Start connection monitorn.startConnectionMonitor(ctx, n.config.MonitorBootstrap, n.config.MonitorInterval, 3)+ go p2plimits.ObserveStreamPressure(n.Host.Network().Conns, busiestConnStreams, logger.Warnf).Run(ctx, streamPressureInterval)// Periodically and on-demand reprovide registered services to the DHTn.startReprovideLoop(ctx, ReprovideInterval)diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.goindex 1b2dbaaf..ebd65547 100644--- a/internal/node/node_metrics.go+++ b/internal/node/node_metrics.go@@ -49,8 +49,21 @@ func init() {for _, result := range []string{lookupFound, lookupNotFound, lookupFailed} {peerLookupsTotal.WithLabelValues(result)}+ for _, result := range []string{reachOK, reachRelayRefused, reachDialTimeout, reachNoAddress, reachDialFailed, reachNoAnswer, reachAuthRejected, reachOtherFailure} {+ providerReachTotal.WithLabelValues(result)+ }}+// providerReachTotal counts the attempts a call made to reach a provider+// for its credential, by outcome; every failure is one 502 at the caller.+var providerReachTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "agentmesh_node_provider_reach_total",+ Help: "Attempts to reach a provider for its credential before a call, by outcome: ok, relay_refused, dial_timeout, no_address, dial_failed, no_answer, auth_rejected, other",+ },+ []string{"result"},+)+// The outcome of a DHT lookup for a peer's addresses: found, found// nothing, or the lookup itself failed (an empty routing table, a timeout).const (diff --git a/internal/node/resource_limits.go b/internal/node/resource_limits.goindex 12bef03b..8cb1f931 100644--- a/internal/node/resource_limits.go+++ b/internal/node/resource_limits.go@@ -16,6 +16,7 @@ package nodeimport ("slices"+ "time""github.com/google/agentmesh/api""github.com/google/agentmesh/internal/p2plimits"@@ -24,6 +25,7 @@ import ("github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/protocol"rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto""github.com/libp2p/go-libp2p/p2p/protocol/identify""github.com/prometheus/client_golang/prometheus""github.com/prometheus/client_golang/prometheus/promauto"@@ -108,6 +110,16 @@ func nodeLimits(conns int) rcmgr.ConcreteLimitConfig {overrides.Protocol[p] = p2plimits.Streams(conns)}overrides.Protocol[api.MCPProtocolID] = p2plimits.Streams(streamsPerConn * conns)++ // A caller that arrives through a router is one relay stop stream, and+ // every caller through the same router is a stream from that one peer.+ // libp2p's default is 128 per peer, which capped a provider at 128+ // callers through each of its routers, so the stop protocol gets the+ // whole connection budget, per peer and in all.+ stop := p2plimits.Streams(conns)+ stop.Memory = rcmgr.LimitVal64(int64(conns) * 64 << 10)+ overrides.Protocol[circuit.ProtoIDv2Stop] = stop+ overrides.ProtocolPeer = map[protocol.ID]rcmgr.ResourceLimits{circuit.ProtoIDv2Stop: stop}return overrides.Build(scaled)}@@ -119,6 +131,21 @@ var streamsRefusedTotal = promauto.NewCounterVec([]string{"scope", "protocol"},)+// busiestConnStreams is the most streams any one connection carries. On a+// provider the connection to a router carries one stream per caller+// relayed through it; a QUIC connection admits+// p2plimits.QUICStreamsPerConnection and the next caller waits.+var busiestConnStreams = promauto.NewGaugeVec(+ prometheus.GaugeOpts{+ Name: "agentmesh_node_busiest_connection_streams",+ Help: "Streams on the connection that carries the most, by transport; QUIC admits 256 per connection and a further stream waits",+ },+ []string{"transport"},+)++// streamPressureInterval is how often connections are sampled for it.+const streamPressureInterval = 15 * time.Second+// newResourceManager is the resource manager the node's host runs with:// nodeLimits, the per-source cap on direct callers, and every refusal// counted and logged. A caller that arrives through a router is notdiff --git a/internal/node/resource_limits_test.go b/internal/node/resource_limits_test.goindex 5542233b..966bbaa3 100644--- a/internal/node/resource_limits_test.go+++ b/internal/node/resource_limits_test.go@@ -21,6 +21,7 @@ import (pubsub "github.com/libp2p/go-libp2p-pubsub""github.com/libp2p/go-libp2p/core/connmgr"rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto""github.com/libp2p/go-libp2p/p2p/protocol/identify")@@ -46,6 +47,9 @@ func TestNodeLimitsFollowTheConnectionBudget(t *testing.T) {for _, p := range pubsub.GossipSubDefaultProtocols {want("gossip protocol "+string(p)+" inbound streams", cfg.Protocol[p].StreamsInbound, ConnHighWaterMark)}+ // Every caller through one router is a stop stream from that one peer.+ want("relay stop protocol inbound streams", cfg.Protocol[circuit.ProtoIDv2Stop].StreamsInbound, ConnHighWaterMark)+ want("relay stop protocol per-peer inbound streams", cfg.ProtocolPeer[circuit.ProtoIDv2Stop].StreamsInbound, ConnHighWaterMark)if ConnsPerSourceIP != ConnHighWaterMark/4 {t.Errorf("ConnsPerSourceIP = %d, want a quarter of the high watermark, %d", ConnsPerSourceIP, ConnHighWaterMark/4)diff --git a/internal/p2plimits/pressure.go b/internal/p2plimits/pressure.gonew file mode 100644index 00000000..d906646d--- /dev/null+++ b/internal/p2plimits/pressure.go@@ -0,0 +1,144 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package p2plimits++import (+ "context"+ "sync"+ "time"++ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/prometheus/client_golang/prometheus"+)++// QUICStreamsPerConnection is how many streams the far end may hold open on+// one QUIC connection, as go-libp2p configures quic-go; a stream opened+// past it waits for one to close, with no error at either end. yamux over+// TCP has no limit of its own; the resource manager's are the only ones.+// Every relayed connection to a peer is one stream on the relay's+// connection to it, so a service called through a router has at most this+// many relayed callers through it over QUIC.+const QUICStreamsPerConnection = 256++// StreamPressureWarnAt is the stream count at which a QUIC connection is+// reported: nine tenths of what it admits.+const StreamPressureWarnAt = QUICStreamsPerConnection * 9 / 10++// TransportQUIC is the transport name libp2p gives a QUIC connection;+// WebTransport runs over QUIC with the same stream limit.+const TransportQUIC = "quic-v1"++// quicTransports are the transports with QUIC's per-connection stream limit.+var quicTransports = map[string]bool{TransportQUIC: true, "quic": true, "webtransport": true}++// StreamPressure reports the connection of a host that carries the most+// streams, by transport, and warns once per peer per RefusalLogInterval when+// a QUIC connection is within reach of its stream limit: the next stream+// between the two waits, which a relay turns into a hop that hangs until+// its connect timeout and a caller into a dial that times out, neither of+// which says why.+type StreamPressure struct {+ conns func() []network.Conn+ busiest *prometheus.GaugeVec+ warnf func(format string, args ...any)+ mu sync.Mutex+ lastLogAt map[peer.ID]time.Time+ now func() time.Time+}++// connLoad is what the sampler reads off a connection.+type connLoad struct {+ peer peer.ID+ transport string+ streams int+}++// ObserveStreamPressure returns a sampler over conns. busiest carries the+// label transport; a nil warnf only samples.+func ObserveStreamPressure(conns func() []network.Conn, busiest *prometheus.GaugeVec, warnf func(format string, args ...any)) *StreamPressure {+ if warnf == nil {+ warnf = func(string, ...any) {}+ }+ return &StreamPressure{conns: conns, busiest: busiest, warnf: warnf, lastLogAt: map[peer.ID]time.Time{}, now: time.Now}+}++// SetClock replaces the clock the log limiter reads; for tests.+func (p *StreamPressure) SetClock(now func() time.Time) { p.now = now }++// Run samples every interval until ctx ends.+func (p *StreamPressure) Run(ctx context.Context, interval time.Duration) {+ ticker := time.NewTicker(interval)+ defer ticker.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ p.Sample()+ }+ }+}++// Sample reads every connection once.+func (p *StreamPressure) Sample() {+ conns := p.conns()+ loads := make([]connLoad, 0, len(conns))+ for _, c := range conns {+ loads = append(loads, connLoad{peer: c.RemotePeer(), transport: c.ConnState().Transport, streams: c.Stat().NumStreams})+ }+ p.observe(loads)+}++func (p *StreamPressure) observe(loads []connLoad) {+ // Relayed connections are streams on a transport connection already+ // counted; they carry one stream each and have no limit of their own.+ busiest := map[string]int{TransportQUIC: 0, "tcp": 0}+ for _, l := range loads {+ if l.transport == "p2p-circuit" {+ continue+ }+ if l.streams > busiest[l.transport] {+ busiest[l.transport] = l.streams+ }+ }+ for transport, n := range busiest {+ p.busiest.WithLabelValues(transport).Set(float64(n))+ }++ threshold := StreamPressureWarnAt+ now := p.now()+ p.mu.Lock()+ for k, t := range p.lastLogAt {+ if now.Sub(t) >= RefusalLogInterval {+ delete(p.lastLogAt, k)+ }+ }+ var report []connLoad+ for _, l := range loads {+ if !quicTransports[l.transport] || l.streams < threshold {+ continue+ }+ if _, recent := p.lastLogAt[l.peer]; recent {+ continue+ }+ p.lastLogAt[l.peer] = now+ report = append(report, l)+ }+ p.mu.Unlock()+ for _, l := range report {+ p.warnf("[Conn] Connection to %s over %s carries %d streams of the %d QUIC admits per connection; a further stream between the two waits for one to close, which a relayed connection shows as a dial that times out", l.peer, l.transport, l.streams, QUICStreamsPerConnection)+ }+}diff --git a/internal/p2plimits/pressure_test.go b/internal/p2plimits/pressure_test.gonew file mode 100644index 00000000..ed06f9cf--- /dev/null+++ b/internal/p2plimits/pressure_test.go@@ -0,0 +1,92 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package p2plimits++import (+ "fmt"+ "strings"+ "testing"+ "time"++ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/test"+ "github.com/prometheus/client_golang/prometheus"+ dto "github.com/prometheus/client_model/go"+)++func gaugeValue(t *testing.T, g prometheus.Gauge) float64 {+ t.Helper()+ var m dto.Metric+ if err := g.Write(&m); err != nil {+ t.Fatal(err)+ }+ return m.GetGauge().GetValue()+}++// The busiest connection per transport is a gauge, and a QUIC connection+// near its stream limit is named once per interval: a relay's connection+// to a service holds one stream per relayed caller, and the 257th caller+// over QUIC waits with no error at either end.+func TestStreamPressureNamesAQUICConnectionNearItsLimit(t *testing.T) {+ relay, other := test.RandPeerIDFatal(t), test.RandPeerIDFatal(t)+ loads := []connLoad{+ {peer: relay, transport: TransportQUIC, streams: 240},+ {peer: other, transport: TransportQUIC, streams: 3},+ {peer: test.RandPeerIDFatal(t), transport: "tcp", streams: 500},+ }+ busiest := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_busiest"}, []string{"transport"})+ var lines []string+ now := time.Unix(1_700_000_000, 0)+ p := ObserveStreamPressure(func() []network.Conn { return nil }, busiest, func(format string, args ...any) {+ lines = append(lines, fmt.Sprintf(format, args...))+ })+ p.SetClock(func() time.Time { return now })++ p.observe(loads)+ if got := gaugeValue(t, busiest.WithLabelValues(TransportQUIC)); got != 240 {+ t.Errorf("busiest quic = %v, want 240", got)+ }+ if got := gaugeValue(t, busiest.WithLabelValues("tcp")); got != 500 {+ t.Errorf("busiest tcp = %v, want 500: tcp has no stream limit, the gauge still shows the load", got)+ }+ if len(lines) != 1 || !strings.Contains(lines[0], relay.String()) || !strings.Contains(lines[0], "240 streams of the 256") {+ t.Fatalf("warnings = %q, want one naming the relay's connection", lines)+ }++ // Still loaded a sample later: not repeated within the interval.+ p.observe(loads)+ if len(lines) != 1 {+ t.Errorf("warned again within the interval: %q", lines)+ }+ now = now.Add(RefusalLogInterval)+ p.observe(loads)+ if len(lines) != 2 {+ t.Errorf("not warned again after the interval: %q", lines)+ }++ // Below the threshold nothing is said, and a peer that is quiet for an+ // interval is forgotten.+ now = now.Add(RefusalLogInterval)+ p.observe([]connLoad{{peer: relay, transport: TransportQUIC, streams: 10}})+ if len(lines) != 2 {+ t.Errorf("warned for a connection under the threshold: %q", lines[len(lines)-1])+ }+ if got := gaugeValue(t, busiest.WithLabelValues(TransportQUIC)); got != 10 {+ t.Errorf("busiest quic after the load left = %v, want 10", got)+ }+ if n := len(p.lastLogAt); n != 0 {+ t.Errorf("peers remembered after an interval of quiet = %d, want 0", n)+ }+}diff --git a/internal/p2plimits/streams.go b/internal/p2plimits/streams.goindex 7e7e124e..d3c7e4b7 100644--- a/internal/p2plimits/streams.go+++ b/internal/p2plimits/streams.go@@ -87,7 +87,11 @@ func (s *StreamRefusals) refusedStream(scope, name string) {}s.mu.Unlock()if shouldLog {- s.warnf("[Stream] Refused %d stream(s) on %s at the %s scope since the last report; the peer sees its stream reset", n, name, scope)+ what := "at the " + scope + " scope"+ if name != "" {+ what = "on " + name + " " + what+ }+ s.warnf("[Stream] Refused %d stream(s) %s since the last report; the peer sees its stream reset", n, what)}}diff --git a/internal/p2plimits/streams_test.go b/internal/p2plimits/streams_test.goindex a5af44e1..38b74bb6 100644--- a/internal/p2plimits/streams_test.go+++ b/internal/p2plimits/streams_test.go@@ -90,4 +90,9 @@ func TestStreamRefusalsLogOneProtocolOncePerInterval(t *testing.T) {if len(lines) != 4 || lines[3] != "[Stream] Refused 3 stream(s) on /a at the protocol scope since the last report; the peer sees its stream reset" {t.Errorf("after the interval: %q", lines[len(lines)-1])}+ // A scope with no protocol to name is said without one.+ s.BlockPeer("")+ if lines[len(lines)-1] != "[Stream] Refused 1 stream(s) at the peer scope since the last report; the peer sees its stream reset" {+ t.Errorf("peer scope: %q", lines[len(lines)-1])+ }}diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.goindex a8b31b47..6f724a01 100644--- a/internal/router/resource_observer.go+++ b/internal/router/resource_observer.go@@ -40,6 +40,21 @@ var streamsRefusedTotal = promauto.NewCounterVec([]string{"scope", "protocol"},)+// busiestConnStreams is the most streams any one connection carries. A+// QUIC connection admits p2plimits.QUICStreamsPerConnection; the router's+// connection to a service it relays for carries one stream per relayed+// caller, so this reaching that figure is callers waiting on the relay.+var busiestConnStreams = promauto.NewGaugeVec(+ prometheus.GaugeOpts{+ Name: "agentmesh_router_busiest_connection_streams",+ Help: "Streams on the connection that carries the most, by transport; QUIC admits 256 per connection and a further stream waits",+ },+ []string{"transport"},+)++// streamPressureInterval is how often connections are sampled for it.+const streamPressureInterval = 15 * time.Second+func init() {for _, v := range p2plimits.Reasons {inboundRefusedTotal.WithLabelValues(v)diff --git a/internal/router/router.go b/internal/router/router.goindex 167f9551..235f64af 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -572,13 +572,17 @@ func (r *Router) Start() (retErr error) {})// 5. Start background routines- r.wg.Add(6)+ r.wg.Add(7)go r.runLeaseRenewalLoop()go r.runKeysSyncLoop()go r.runFederationLoop()go r.runBiscuitRenewalLoop()go r.runShedLoop()go r.listenForControlPlaneEvents(r.ctx)+ go func() {+ defer r.wg.Done()+ p2plimits.ObserveStreamPressure(hostNode.Network().Conns, busiestConnStreams, logger.Warnf).Run(r.ctx, streamPressureInterval)+ }()r.isReady.Store(true)logger.Infof("Router Online. PeerID: %s, ListenAddrs: %v", r.Host.ID(), r.Host.Addrs())diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.mdindex 707bdb5c..f93c8e0d 100644--- a/site/content/docs/guides/scalability.md+++ b/site/content/docs/guides/scalability.md@@ -237,6 +237,8 @@ The router and the control plane serve Prometheus metrics on| `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: direct callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `rate_limit` a burst from one address, `scope_limit` more callers at once than the node's connection budget, relayed ones included. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. || `agentmesh_node_transit_handshakes_total{router,result}` | On a caller: handshakes with routers it did not hold, run to reach a peer through them. `router="unlisted"` is a router the node's catalog missed, which a member that started during a router rollout produces until its next sync; `result="failed"` rising is a router that refuses the node or is gone while peers still announce circuits through it. || `agentmesh_node_peer_lookups_total{result}` | On a caller: DHT lookups for a peer it held no address for. `not_found` is a peer the DHT does not carry, and the call falls back to circuits through the caller's own routers, which reach the peer only if it holds one of them; `failed` rising is the caller cut off from the DHT (`agentmesh_node_dht_routing_table_size` at 0). |+| `agentmesh_node_provider_reach_total{result}` | On a caller: attempts to reach a provider for its credential before a call; every failure is one 502 at the caller. `relay_refused` is a router that has not admitted the caller or the provider, `dial_timeout` a provider gone or a relay connection at its stream limit (below), `no_address` a provider the directory does not carry. |+| `agentmesh_router_busiest_connection_streams{transport}`, `agentmesh_node_busiest_connection_streams{transport}` | The most streams any one connection carries. Every relayed connection is one stream on the relay's connection to each end, and a QUIC connection admits 256 streams: a provider reached through one router over QUIC has at most 256 callers through it at once, and the 257th waits until one leaves, which it sees as a dial that times out. Both ends log the connection once a minute while it is within a tenth of the limit. A provider that expects more callers than that listens on TCP as well, or holds more routers. || `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. || `libp2p_relaysvc_connection_rejections_total{reason="resource limit exceeded"}` | A service has more callers than `--relay-max-circuits`. || `libp2p_rcmgr_blocked_resources` | libp2p refused a connection or stream on a scope. On a router this should stay flat. |diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex a03720fb..acaba254 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -122,6 +122,7 @@ The metrics address exposes, next to the Go runtime metrics:| `agentmesh_router_unauthenticated_connections_closed_total` | Connections closed at `--auth-deadline` without a handshake. A steady rate is DHT traffic from members attached to other routers; a burst is a scanner. || `agentmesh_router_inbound_connections_refused_total` | Inbound connections refused by the per-address cap. || `agentmesh_router_streams_refused_total{scope,protocol}` | Streams the resource manager refused, by scope and protocol or service. A count on `/mesh/kad/1.0.0` or a `/meshsub/*` protocol is members the router keeps out of discovery; see the scalability guide. |+| `agentmesh_router_busiest_connection_streams{transport}` | The most streams any one connection carries. The connection to a service the router relays for holds one stream per relayed caller, and QUIC admits 256 per connection; see the scalability guide. || `libp2p_relaysvc_*` | The relay service's own counters: `reservations_total{type}` and `connections_total{type}` (`opened`, `closed`, `renewed`), requests by response status, rejections and bytes relayed. The difference between `opened` and `closed` is the number in use; read it against `agentmesh_router_relay_limit` to see how close the relay is to its budget. |A fleet is at capacity when `agentmesh_router_connections{direction="inbound"}`