Skip to content

Agent Mesh

node, router: name the connection at QUIC's stream limit, count how a caller fares reaching a provider

Antonio Ojeae46a2c038713
diff --git a/internal/node/labels_gate.go b/internal/node/labels_gate.go
index 0f5164d3..293e9bf2 100644
--- a/internal/node/labels_gate.go
+++ b/internal/node/labels_gate.go
@@ -28,6 +28,7 @@ import (
"github.com/google/agentmesh/internal/identity"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/p2p/net/swarm"
"github.com/libp2p/go-msgio"
"google.golang.org/protobuf/proto"
)
@@ -98,6 +99,48 @@ const (
// is the mesh's state and a caller may retry it; the second is a verdict.
var ErrProviderUnreachable = errors.New("provider unreachable")
+// How reaching a provider for its credential went; the label values of
+// providerReachTotal. The caller sees one 502 for all of the failures, so
+// the counter tells them apart: the relay refusing the circuit, the dial
+// timing out (a relay's connection to the provider at its stream limit,
+// or a provider gone), no address to dial, the provider closing the
+// stream before answering.
+const (
+ reachOK = "ok"
+ reachRelayRefused = "relay_refused"
+ reachDialTimeout = "dial_timeout"
+ reachNoAddress = "no_address"
+ reachDialFailed = "dial_failed"
+ reachNoAnswer = "no_answer"
+ reachAuthRejected = "auth_rejected"
+ reachOtherFailure = "other"
+ relayRefusedMarker = "error opening relay circuit"
+)
+
+// reachFailure classifies why a provider was not reached.
+func reachFailure(err error) string {
+ var dialErr *swarm.DialError
+ switch {
+ case errors.As(err, &dialErr):
+ switch {
+ case dialErr.Timeout(), errors.Is(err, context.DeadlineExceeded):
+ return reachDialTimeout
+ case errors.Is(err, swarm.ErrNoAddresses), errors.Is(err, swarm.ErrNoGoodAddresses):
+ return reachNoAddress
+ case strings.Contains(err.Error(), relayRefusedMarker):
+ return reachRelayRefused
+ }
+ return reachDialFailed
+ case errors.Is(err, context.DeadlineExceeded):
+ return reachDialTimeout
+ case strings.Contains(err.Error(), "read auth response"):
+ return reachNoAnswer
+ case strings.Contains(err.Error(), "auth rejected"):
+ return reachAuthRejected
+ }
+ return reachOtherFailure
+}
+
// labelGateStatus is the HTTP answer for a label-gate failure, in the
// order http.Error takes it: 502 when the provider was never heard from, so
// a caller does not read a dead or busy destination as a denial of its
@@ -174,8 +217,10 @@ func (n *AgentMeshNode) VerifyPeerLabels(ctx context.Context, peerID peer.ID, re
providerBiscuit, err := n.fetchPeerBiscuit(ctx, peerID)
if err != nil {
+ providerReachTotal.WithLabelValues(reachFailure(err)).Inc()
return fmt.Errorf("provider %s labels unverifiable: %w: %w", peerID, ErrProviderUnreachable, err)
}
+ providerReachTotal.WithLabelValues(reachOK).Inc()
if err := n.checkPeerLabels(providerBiscuit, peerID, required); err != nil {
return err
}
diff --git a/internal/node/labels_gate_test.go b/internal/node/labels_gate_test.go
index d2f30720..5ef50921 100644
--- a/internal/node/labels_gate_test.go
+++ b/internal/node/labels_gate_test.go
@@ -18,6 +18,7 @@ import (
"context"
"crypto/ed25519"
"errors"
+ "fmt"
"net/http"
"testing"
"time"
@@ -27,6 +28,8 @@ import (
lru "github.com/hashicorp/golang-lru/v2"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/p2p/net/swarm"
+ "github.com/multiformats/go-multiaddr"
)
// The gate guards calls that ride relayed connections (egress proxy, MCP
@@ -329,3 +332,30 @@ func TestLabelGateKeyIsUnambiguousWithSeparatorsInValues(t *testing.T) {
}
}
}
+
+// A provider the caller did not reach is one 502 whatever happened; the
+// counter tells the cases apart from the error libp2p hands back.
+func TestReachFailureClassifiesWhatKeptTheProviderAway(t *testing.T) {
+ peerID := peer.ID("12D3KooWtest")
+ relayRefused := &swarm.DialError{Peer: peerID, Cause: swarm.ErrAllDialsFailed, DialErrors: []swarm.TransportError{
+ {Address: multiaddr.StringCast("/p2p-circuit"), Cause: errors.New("error opening relay circuit: PERMISSION_DENIED (202)")},
+ }}
+ timedOut := &swarm.DialError{Peer: peerID, Cause: context.DeadlineExceeded}
+ for _, tc := range []struct {
+ err error
+ want string
+ }{
+ {fmt.Errorf("failed to open auth stream: %w", relayRefused), reachRelayRefused},
+ {fmt.Errorf("failed to open auth stream: %w", timedOut), reachDialTimeout},
+ {fmt.Errorf("failed to open auth stream: %w", &swarm.DialError{Peer: peerID, Cause: swarm.ErrNoAddresses}), reachNoAddress},
+ {fmt.Errorf("failed to open auth stream: %w", &swarm.DialError{Peer: peerID, Cause: swarm.ErrAllDialsFailed, DialErrors: []swarm.TransportError{{Cause: errors.New("connection refused")}}}), reachDialFailed},
+ {fmt.Errorf("failed to open auth stream: %w", context.DeadlineExceeded), reachDialTimeout},
+ {errors.New("read auth response (peer may predate mutual auth): EOF"), reachNoAnswer},
+ {errors.New("auth rejected: expired"), reachAuthRejected},
+ {errors.New("something else"), reachOtherFailure},
+ } {
+ if got := reachFailure(tc.err); got != tc.want {
+ t.Errorf("reachFailure(%v) = %s, want %s", tc.err, got, tc.want)
+ }
+ }
+}
diff --git a/internal/node/node.go b/internal/node/node.go
index 5e6eb174..946f01cd 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -42,6 +42,7 @@ import (
cpclient "github.com/google/agentmesh/internal/controlplane/client"
"github.com/google/agentmesh/internal/identity"
samdiscovery "github.com/google/agentmesh/internal/node/discovery"
+ "github.com/google/agentmesh/internal/p2plimits"
"github.com/google/agentmesh/internal/ratelimit"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/ipfs/go-cid"
@@ -799,6 +800,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
// Start connection monitor
n.startConnectionMonitor(ctx, n.config.MonitorBootstrap, n.config.MonitorInterval, 3)
+ go p2plimits.ObserveStreamPressure(n.Host.Network().Conns, busiestConnStreams, logger.Warnf).Run(ctx, streamPressureInterval)
// Periodically and on-demand reprovide registered services to the DHT
n.startReprovideLoop(ctx, ReprovideInterval)
diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.go
index 1b2dbaaf..ebd65547 100644
--- a/internal/node/node_metrics.go
+++ b/internal/node/node_metrics.go
@@ -49,8 +49,21 @@ func init() {
for _, result := range []string{lookupFound, lookupNotFound, lookupFailed} {
peerLookupsTotal.WithLabelValues(result)
}
+ for _, result := range []string{reachOK, reachRelayRefused, reachDialTimeout, reachNoAddress, reachDialFailed, reachNoAnswer, reachAuthRejected, reachOtherFailure} {
+ providerReachTotal.WithLabelValues(result)
+ }
}
+// providerReachTotal counts the attempts a call made to reach a provider
+// for its credential, by outcome; every failure is one 502 at the caller.
+var providerReachTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_node_provider_reach_total",
+ Help: "Attempts to reach a provider for its credential before a call, by outcome: ok, relay_refused, dial_timeout, no_address, dial_failed, no_answer, auth_rejected, other",
+ },
+ []string{"result"},
+)
+
// The outcome of a DHT lookup for a peer's addresses: found, found
// nothing, or the lookup itself failed (an empty routing table, a timeout).
const (
diff --git a/internal/node/resource_limits.go b/internal/node/resource_limits.go
index 12bef03b..8cb1f931 100644
--- a/internal/node/resource_limits.go
+++ b/internal/node/resource_limits.go
@@ -16,6 +16,7 @@ package node
import (
"slices"
+ "time"
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/p2plimits"
@@ -24,6 +25,7 @@ import (
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/protocol"
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
"github.com/libp2p/go-libp2p/p2p/protocol/identify"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@@ -108,6 +110,16 @@ func nodeLimits(conns int) rcmgr.ConcreteLimitConfig {
overrides.Protocol[p] = p2plimits.Streams(conns)
}
overrides.Protocol[api.MCPProtocolID] = p2plimits.Streams(streamsPerConn * conns)
+
+ // A caller that arrives through a router is one relay stop stream, and
+ // every caller through the same router is a stream from that one peer.
+ // libp2p's default is 128 per peer, which capped a provider at 128
+ // callers through each of its routers, so the stop protocol gets the
+ // whole connection budget, per peer and in all.
+ stop := p2plimits.Streams(conns)
+ stop.Memory = rcmgr.LimitVal64(int64(conns) * 64 << 10)
+ overrides.Protocol[circuit.ProtoIDv2Stop] = stop
+ overrides.ProtocolPeer = map[protocol.ID]rcmgr.ResourceLimits{circuit.ProtoIDv2Stop: stop}
return overrides.Build(scaled)
}
@@ -119,6 +131,21 @@ var streamsRefusedTotal = promauto.NewCounterVec(
[]string{"scope", "protocol"},
)
+// busiestConnStreams is the most streams any one connection carries. On a
+// provider the connection to a router carries one stream per caller
+// relayed through it; a QUIC connection admits
+// p2plimits.QUICStreamsPerConnection and the next caller waits.
+var busiestConnStreams = promauto.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Name: "agentmesh_node_busiest_connection_streams",
+ Help: "Streams on the connection that carries the most, by transport; QUIC admits 256 per connection and a further stream waits",
+ },
+ []string{"transport"},
+)
+
+// streamPressureInterval is how often connections are sampled for it.
+const streamPressureInterval = 15 * time.Second
+
// newResourceManager is the resource manager the node's host runs with:
// nodeLimits, the per-source cap on direct callers, and every refusal
// counted and logged. A caller that arrives through a router is not
diff --git a/internal/node/resource_limits_test.go b/internal/node/resource_limits_test.go
index 5542233b..966bbaa3 100644
--- a/internal/node/resource_limits_test.go
+++ b/internal/node/resource_limits_test.go
@@ -21,6 +21,7 @@ import (
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/connmgr"
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
"github.com/libp2p/go-libp2p/p2p/protocol/identify"
)
@@ -46,6 +47,9 @@ func TestNodeLimitsFollowTheConnectionBudget(t *testing.T) {
for _, p := range pubsub.GossipSubDefaultProtocols {
want("gossip protocol "+string(p)+" inbound streams", cfg.Protocol[p].StreamsInbound, ConnHighWaterMark)
}
+ // Every caller through one router is a stop stream from that one peer.
+ want("relay stop protocol inbound streams", cfg.Protocol[circuit.ProtoIDv2Stop].StreamsInbound, ConnHighWaterMark)
+ want("relay stop protocol per-peer inbound streams", cfg.ProtocolPeer[circuit.ProtoIDv2Stop].StreamsInbound, ConnHighWaterMark)
if ConnsPerSourceIP != ConnHighWaterMark/4 {
t.Errorf("ConnsPerSourceIP = %d, want a quarter of the high watermark, %d", ConnsPerSourceIP, ConnHighWaterMark/4)
diff --git a/internal/p2plimits/pressure.go b/internal/p2plimits/pressure.go
new file mode 100644
index 00000000..d906646d
--- /dev/null
+++ b/internal/p2plimits/pressure.go
@@ -0,0 +1,144 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package p2plimits
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+// QUICStreamsPerConnection is how many streams the far end may hold open on
+// one QUIC connection, as go-libp2p configures quic-go; a stream opened
+// past it waits for one to close, with no error at either end. yamux over
+// TCP has no limit of its own; the resource manager's are the only ones.
+// Every relayed connection to a peer is one stream on the relay's
+// connection to it, so a service called through a router has at most this
+// many relayed callers through it over QUIC.
+const QUICStreamsPerConnection = 256
+
+// StreamPressureWarnAt is the stream count at which a QUIC connection is
+// reported: nine tenths of what it admits.
+const StreamPressureWarnAt = QUICStreamsPerConnection * 9 / 10
+
+// TransportQUIC is the transport name libp2p gives a QUIC connection;
+// WebTransport runs over QUIC with the same stream limit.
+const TransportQUIC = "quic-v1"
+
+// quicTransports are the transports with QUIC's per-connection stream limit.
+var quicTransports = map[string]bool{TransportQUIC: true, "quic": true, "webtransport": true}
+
+// StreamPressure reports the connection of a host that carries the most
+// streams, by transport, and warns once per peer per RefusalLogInterval when
+// a QUIC connection is within reach of its stream limit: the next stream
+// between the two waits, which a relay turns into a hop that hangs until
+// its connect timeout and a caller into a dial that times out, neither of
+// which says why.
+type StreamPressure struct {
+ conns func() []network.Conn
+ busiest *prometheus.GaugeVec
+ warnf func(format string, args ...any)
+ mu sync.Mutex
+ lastLogAt map[peer.ID]time.Time
+ now func() time.Time
+}
+
+// connLoad is what the sampler reads off a connection.
+type connLoad struct {
+ peer peer.ID
+ transport string
+ streams int
+}
+
+// ObserveStreamPressure returns a sampler over conns. busiest carries the
+// label transport; a nil warnf only samples.
+func ObserveStreamPressure(conns func() []network.Conn, busiest *prometheus.GaugeVec, warnf func(format string, args ...any)) *StreamPressure {
+ if warnf == nil {
+ warnf = func(string, ...any) {}
+ }
+ return &StreamPressure{conns: conns, busiest: busiest, warnf: warnf, lastLogAt: map[peer.ID]time.Time{}, now: time.Now}
+}
+
+// SetClock replaces the clock the log limiter reads; for tests.
+func (p *StreamPressure) SetClock(now func() time.Time) { p.now = now }
+
+// Run samples every interval until ctx ends.
+func (p *StreamPressure) Run(ctx context.Context, interval time.Duration) {
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ p.Sample()
+ }
+ }
+}
+
+// Sample reads every connection once.
+func (p *StreamPressure) Sample() {
+ conns := p.conns()
+ loads := make([]connLoad, 0, len(conns))
+ for _, c := range conns {
+ loads = append(loads, connLoad{peer: c.RemotePeer(), transport: c.ConnState().Transport, streams: c.Stat().NumStreams})
+ }
+ p.observe(loads)
+}
+
+func (p *StreamPressure) observe(loads []connLoad) {
+ // Relayed connections are streams on a transport connection already
+ // counted; they carry one stream each and have no limit of their own.
+ busiest := map[string]int{TransportQUIC: 0, "tcp": 0}
+ for _, l := range loads {
+ if l.transport == "p2p-circuit" {
+ continue
+ }
+ if l.streams > busiest[l.transport] {
+ busiest[l.transport] = l.streams
+ }
+ }
+ for transport, n := range busiest {
+ p.busiest.WithLabelValues(transport).Set(float64(n))
+ }
+
+ threshold := StreamPressureWarnAt
+ now := p.now()
+ p.mu.Lock()
+ for k, t := range p.lastLogAt {
+ if now.Sub(t) >= RefusalLogInterval {
+ delete(p.lastLogAt, k)
+ }
+ }
+ var report []connLoad
+ for _, l := range loads {
+ if !quicTransports[l.transport] || l.streams < threshold {
+ continue
+ }
+ if _, recent := p.lastLogAt[l.peer]; recent {
+ continue
+ }
+ p.lastLogAt[l.peer] = now
+ report = append(report, l)
+ }
+ p.mu.Unlock()
+ for _, l := range report {
+ p.warnf("[Conn] Connection to %s over %s carries %d streams of the %d QUIC admits per connection; a further stream between the two waits for one to close, which a relayed connection shows as a dial that times out", l.peer, l.transport, l.streams, QUICStreamsPerConnection)
+ }
+}
diff --git a/internal/p2plimits/pressure_test.go b/internal/p2plimits/pressure_test.go
new file mode 100644
index 00000000..ed06f9cf
--- /dev/null
+++ b/internal/p2plimits/pressure_test.go
@@ -0,0 +1,92 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package p2plimits
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/test"
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
+)
+
+func gaugeValue(t *testing.T, g prometheus.Gauge) float64 {
+ t.Helper()
+ var m dto.Metric
+ if err := g.Write(&m); err != nil {
+ t.Fatal(err)
+ }
+ return m.GetGauge().GetValue()
+}
+
+// The busiest connection per transport is a gauge, and a QUIC connection
+// near its stream limit is named once per interval: a relay's connection
+// to a service holds one stream per relayed caller, and the 257th caller
+// over QUIC waits with no error at either end.
+func TestStreamPressureNamesAQUICConnectionNearItsLimit(t *testing.T) {
+ relay, other := test.RandPeerIDFatal(t), test.RandPeerIDFatal(t)
+ loads := []connLoad{
+ {peer: relay, transport: TransportQUIC, streams: 240},
+ {peer: other, transport: TransportQUIC, streams: 3},
+ {peer: test.RandPeerIDFatal(t), transport: "tcp", streams: 500},
+ }
+ busiest := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_busiest"}, []string{"transport"})
+ var lines []string
+ now := time.Unix(1_700_000_000, 0)
+ p := ObserveStreamPressure(func() []network.Conn { return nil }, busiest, func(format string, args ...any) {
+ lines = append(lines, fmt.Sprintf(format, args...))
+ })
+ p.SetClock(func() time.Time { return now })
+
+ p.observe(loads)
+ if got := gaugeValue(t, busiest.WithLabelValues(TransportQUIC)); got != 240 {
+ t.Errorf("busiest quic = %v, want 240", got)
+ }
+ if got := gaugeValue(t, busiest.WithLabelValues("tcp")); got != 500 {
+ t.Errorf("busiest tcp = %v, want 500: tcp has no stream limit, the gauge still shows the load", got)
+ }
+ if len(lines) != 1 || !strings.Contains(lines[0], relay.String()) || !strings.Contains(lines[0], "240 streams of the 256") {
+ t.Fatalf("warnings = %q, want one naming the relay's connection", lines)
+ }
+
+ // Still loaded a sample later: not repeated within the interval.
+ p.observe(loads)
+ if len(lines) != 1 {
+ t.Errorf("warned again within the interval: %q", lines)
+ }
+ now = now.Add(RefusalLogInterval)
+ p.observe(loads)
+ if len(lines) != 2 {
+ t.Errorf("not warned again after the interval: %q", lines)
+ }
+
+ // Below the threshold nothing is said, and a peer that is quiet for an
+ // interval is forgotten.
+ now = now.Add(RefusalLogInterval)
+ p.observe([]connLoad{{peer: relay, transport: TransportQUIC, streams: 10}})
+ if len(lines) != 2 {
+ t.Errorf("warned for a connection under the threshold: %q", lines[len(lines)-1])
+ }
+ if got := gaugeValue(t, busiest.WithLabelValues(TransportQUIC)); got != 10 {
+ t.Errorf("busiest quic after the load left = %v, want 10", got)
+ }
+ if n := len(p.lastLogAt); n != 0 {
+ t.Errorf("peers remembered after an interval of quiet = %d, want 0", n)
+ }
+}
diff --git a/internal/p2plimits/streams.go b/internal/p2plimits/streams.go
index 7e7e124e..d3c7e4b7 100644
--- a/internal/p2plimits/streams.go
+++ b/internal/p2plimits/streams.go
@@ -87,7 +87,11 @@ func (s *StreamRefusals) refusedStream(scope, name string) {
}
s.mu.Unlock()
if shouldLog {
- s.warnf("[Stream] Refused %d stream(s) on %s at the %s scope since the last report; the peer sees its stream reset", n, name, scope)
+ what := "at the " + scope + " scope"
+ if name != "" {
+ what = "on " + name + " " + what
+ }
+ s.warnf("[Stream] Refused %d stream(s) %s since the last report; the peer sees its stream reset", n, what)
}
}
diff --git a/internal/p2plimits/streams_test.go b/internal/p2plimits/streams_test.go
index a5af44e1..38b74bb6 100644
--- a/internal/p2plimits/streams_test.go
+++ b/internal/p2plimits/streams_test.go
@@ -90,4 +90,9 @@ func TestStreamRefusalsLogOneProtocolOncePerInterval(t *testing.T) {
if len(lines) != 4 || lines[3] != "[Stream] Refused 3 stream(s) on /a at the protocol scope since the last report; the peer sees its stream reset" {
t.Errorf("after the interval: %q", lines[len(lines)-1])
}
+ // A scope with no protocol to name is said without one.
+ s.BlockPeer("")
+ if lines[len(lines)-1] != "[Stream] Refused 1 stream(s) at the peer scope since the last report; the peer sees its stream reset" {
+ t.Errorf("peer scope: %q", lines[len(lines)-1])
+ }
}
diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.go
index a8b31b47..6f724a01 100644
--- a/internal/router/resource_observer.go
+++ b/internal/router/resource_observer.go
@@ -40,6 +40,21 @@ var streamsRefusedTotal = promauto.NewCounterVec(
[]string{"scope", "protocol"},
)
+// busiestConnStreams is the most streams any one connection carries. A
+// QUIC connection admits p2plimits.QUICStreamsPerConnection; the router's
+// connection to a service it relays for carries one stream per relayed
+// caller, so this reaching that figure is callers waiting on the relay.
+var busiestConnStreams = promauto.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Name: "agentmesh_router_busiest_connection_streams",
+ Help: "Streams on the connection that carries the most, by transport; QUIC admits 256 per connection and a further stream waits",
+ },
+ []string{"transport"},
+)
+
+// streamPressureInterval is how often connections are sampled for it.
+const streamPressureInterval = 15 * time.Second
+
func init() {
for _, v := range p2plimits.Reasons {
inboundRefusedTotal.WithLabelValues(v)
diff --git a/internal/router/router.go b/internal/router/router.go
index 167f9551..235f64af 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -572,13 +572,17 @@ func (r *Router) Start() (retErr error) {
})
// 5. Start background routines
- r.wg.Add(6)
+ r.wg.Add(7)
go r.runLeaseRenewalLoop()
go r.runKeysSyncLoop()
go r.runFederationLoop()
go r.runBiscuitRenewalLoop()
go r.runShedLoop()
go r.listenForControlPlaneEvents(r.ctx)
+ go func() {
+ defer r.wg.Done()
+ p2plimits.ObserveStreamPressure(hostNode.Network().Conns, busiestConnStreams, logger.Warnf).Run(r.ctx, streamPressureInterval)
+ }()
r.isReady.Store(true)
logger.Infof("Router Online. PeerID: %s, ListenAddrs: %v", r.Host.ID(), r.Host.Addrs())
diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.md
index 707bdb5c..f93c8e0d 100644
--- a/site/content/docs/guides/scalability.md
+++ b/site/content/docs/guides/scalability.md
@@ -237,6 +237,8 @@ The router and the control plane serve Prometheus metrics on
| `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: direct callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `rate_limit` a burst from one address, `scope_limit` more callers at once than the node's connection budget, relayed ones included. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. |
| `agentmesh_node_transit_handshakes_total{router,result}` | On a caller: handshakes with routers it did not hold, run to reach a peer through them. `router="unlisted"` is a router the node's catalog missed, which a member that started during a router rollout produces until its next sync; `result="failed"` rising is a router that refuses the node or is gone while peers still announce circuits through it. |
| `agentmesh_node_peer_lookups_total{result}` | On a caller: DHT lookups for a peer it held no address for. `not_found` is a peer the DHT does not carry, and the call falls back to circuits through the caller's own routers, which reach the peer only if it holds one of them; `failed` rising is the caller cut off from the DHT (`agentmesh_node_dht_routing_table_size` at 0). |
+| `agentmesh_node_provider_reach_total{result}` | On a caller: attempts to reach a provider for its credential before a call; every failure is one 502 at the caller. `relay_refused` is a router that has not admitted the caller or the provider, `dial_timeout` a provider gone or a relay connection at its stream limit (below), `no_address` a provider the directory does not carry. |
+| `agentmesh_router_busiest_connection_streams{transport}`, `agentmesh_node_busiest_connection_streams{transport}` | The most streams any one connection carries. Every relayed connection is one stream on the relay's connection to each end, and a QUIC connection admits 256 streams: a provider reached through one router over QUIC has at most 256 callers through it at once, and the 257th waits until one leaves, which it sees as a dial that times out. Both ends log the connection once a minute while it is within a tenth of the limit. A provider that expects more callers than that listens on TCP as well, or holds more routers. |
| `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. |
| `libp2p_relaysvc_connection_rejections_total{reason="resource limit exceeded"}` | A service has more callers than `--relay-max-circuits`. |
| `libp2p_rcmgr_blocked_resources` | libp2p refused a connection or stream on a scope. On a router this should stay flat. |
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index a03720fb..acaba254 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -122,6 +122,7 @@ The metrics address exposes, next to the Go runtime metrics:
| `agentmesh_router_unauthenticated_connections_closed_total` | Connections closed at `--auth-deadline` without a handshake. A steady rate is DHT traffic from members attached to other routers; a burst is a scanner. |
| `agentmesh_router_inbound_connections_refused_total` | Inbound connections refused by the per-address cap. |
| `agentmesh_router_streams_refused_total{scope,protocol}` | Streams the resource manager refused, by scope and protocol or service. A count on `/mesh/kad/1.0.0` or a `/meshsub/*` protocol is members the router keeps out of discovery; see the scalability guide. |
+| `agentmesh_router_busiest_connection_streams{transport}` | The most streams any one connection carries. The connection to a service the router relays for holds one stream per relayed caller, and QUIC admits 256 per connection; see the scalability guide. |
| `libp2p_relaysvc_*` | The relay service's own counters: `reservations_total{type}` and `connections_total{type}` (`opened`, `closed`, `renewed`), requests by response status, rejections and bytes relayed. The difference between `opened` and `closed` is the number in use; read it against `agentmesh_router_relay_limit` to see how close the relay is to its budget. |
A fleet is at capacity when `agentmesh_router_connections{direction="inbound"}`