Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: export capacity, draining and the relay budget on /metrics
Antonio Ojeaa577440932e9
diff --git a/internal/router/metrics.go b/internal/router/metrics.goindex 21dc5498..39f9c8d0 100644--- a/internal/router/metrics.go+++ b/internal/router/metrics.go@@ -20,6 +20,7 @@ import ("sync""time"+ "github.com/libp2p/go-libp2p/core/network""github.com/prometheus/client_golang/prometheus""github.com/prometheus/client_golang/prometheus/promauto""github.com/prometheus/client_golang/prometheus/promhttp"@@ -81,8 +82,12 @@ type routerStateCollector struct {infoDesc *prometheus.DescreadyDesc *prometheus.Desc+ drainingDesc *prometheus.DescauthenticatedDesc *prometheus.DescconnectedDesc *prometheus.Desc+ connectionsDesc *prometheus.Desc+ watermarkDesc *prometheus.Desc+ relayLimitDesc *prometheus.DescbannedDesc *prometheus.DescdhtDesc *prometheus.DescbiscuitExpiryDesc *prometheus.Desc@@ -99,6 +104,10 @@ func newRouterStateCollector(r *Router) *routerStateCollector {"sam_router_ready","1 once the router is enrolled and its libp2p host is online",nil, nil),+ drainingDesc: prometheus.NewDesc(+ "sam_router_draining",+ "1 once the router has withdrawn its lease and is shutting down",+ nil, nil),authenticatedDesc: prometheus.NewDesc("sam_router_authenticated_peers","Connected peers that have completed the mesh authentication handshake",@@ -107,6 +116,18 @@ func newRouterStateCollector(r *Router) *routerStateCollector {"sam_router_connected_peers","Peers with an open libp2p connection, authenticated or not",nil, nil),+ connectionsDesc: prometheus.NewDesc(+ "sam_router_connections",+ "Open libp2p connections by direction; a peer may hold several",+ []string{"direction"}, nil),+ watermarkDesc: prometheus.NewDesc(+ "sam_router_connection_watermark",+ "Connection manager limits: above high, connections are trimmed down to low",+ []string{"level"}, nil),+ relayLimitDesc: prometheus.NewDesc(+ "sam_router_relay_limit",+ "Relay budgets in force, to read libp2p_relaysvc_* against",+ []string{"limit"}, nil),bannedDesc: prometheus.NewDesc("sam_router_banned_peers","Peers on the ban list synced from the control plane",@@ -124,7 +145,8 @@ func newRouterStateCollector(r *Router) *routerStateCollector {func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {for _, d := range []*prometheus.Desc{- c.infoDesc, c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc, connsPerSourceIPDesc,+ c.infoDesc, c.readyDesc, c.drainingDesc, c.authenticatedDesc, c.connectedDesc, c.connectionsDesc, c.watermarkDesc, c.relayLimitDesc,+ c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc, connsPerSourceIPDesc,} {ch <- d}@@ -133,6 +155,16 @@ func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {r := c.rch <- prometheus.MustNewConstMetric(connsPerSourceIPDesc, prometheus.GaugeValue, float64(effectiveConnsPerSourceIP(r.config.ConnsPerSourceIP)))+ ch <- prometheus.MustNewConstMetric(c.watermarkDesc, prometheus.GaugeValue, float64(r.config.LowWaterMark), "low")+ ch <- prometheus.MustNewConstMetric(c.watermarkDesc, prometheus.GaugeValue, float64(r.config.HighWaterMark), "high")+ ch <- prometheus.MustNewConstMetric(c.relayLimitDesc, prometheus.GaugeValue, float64(r.config.RelayMaxReservations), "reservations")+ ch <- prometheus.MustNewConstMetric(c.relayLimitDesc, prometheus.GaugeValue, float64(r.config.ConnsPerSourceIP), "reservations_per_ip")+ ch <- prometheus.MustNewConstMetric(c.relayLimitDesc, prometheus.GaugeValue, float64(r.config.RelayMaxCircuits), "circuits_per_peer")+ draining := 0.0+ if r.shutdown.Load() {+ draining = 1+ }+ ch <- prometheus.MustNewConstMetric(c.drainingDesc, prometheus.GaugeValue, draining)// isReady is stored after Host and DHT are assigned, so observing it// true is what makes reading them from this goroutine safe.if !r.isReady.Load() {@@ -143,6 +175,16 @@ func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, r.Host.ID().String())ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.authenticatedPeers)))ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(r.Host.Network().Peers())))+ var inbound, outbound int+ for _, conn := range r.Host.Network().Conns() {+ if conn.Stat().Direction == network.DirInbound {+ inbound+++ } else {+ outbound+++ }+ }+ ch <- prometheus.MustNewConstMetric(c.connectionsDesc, prometheus.GaugeValue, float64(inbound), "inbound")+ ch <- prometheus.MustNewConstMetric(c.connectionsDesc, prometheus.GaugeValue, float64(outbound), "outbound")ch <- prometheus.MustNewConstMetric(c.bannedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.bannedPeers)))ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(r.DHT.RoutingTable().Size()))diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.goindex 29877713..7db4b323 100644--- a/internal/router/metrics_test.go+++ b/internal/router/metrics_test.go@@ -101,6 +101,13 @@ func TestRouterMetricsListener(t *testing.T) {`sam_router_info{peer_id="` + host.ID().String() + `"} 1`,// The derived cap: a quarter of the default high watermark."sam_router_conns_per_source_ip_limit 1000",+ "sam_router_draining 0",+ `sam_router_connections{direction="inbound"} 0`,+ `sam_router_connections{direction="outbound"} 0`,+ `sam_router_connection_watermark{level="high"} 4000`,+ `sam_router_relay_limit{limit="reservations"} 4000`,+ `sam_router_relay_limit{limit="reservations_per_ip"} 1000`,+ `sam_router_relay_limit{limit="circuits_per_peer"} 1024`,"sam_router_inbound_connections_refused_total","sam_router_authenticated_peers 2","sam_router_banned_peers 1",diff --git a/internal/router/router.go b/internal/router/router.goindex 49ed49ef..9ea078fe 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -495,10 +495,12 @@ func (r *Router) Start() (retErr error) {return err}- // Setup Relay+ // Setup Relay. Its counters (reservations, circuits, refusals by+ // status) go to the default registry the metrics listener serves._, err = relay.New(hostNode, relay.WithACL(&relayACL{r: r}),relay.WithResources(relayResources(r.config.RelayMaxReservations, r.config.RelayMaxCircuits, r.config.ConnsPerSourceIP)),- relay.WithLimit(relayLimit(r.config.RelayLimitDuration, r.config.RelayLimitData)))+ relay.WithLimit(relayLimit(r.config.RelayLimitDuration, r.config.RelayLimitData)),+ relay.WithMetricsTracer(relay.NewMetricsTracer()))if err != nil {return err}diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex 62fa1e80..c7d53d67 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -76,8 +76,24 @@ important state. Nodes reconnect and publish their services again.## Metrics-The metrics address exposes `sam_router_connected_peers`,-`sam_router_dht_routing_table_size`, `sam_router_auth_handshakes_total` (by-outcome) and `sam_router_lease_renewals_total` (by outcome), next to the Go-runtime metrics. `/healthz` and `/readyz` on the same address are the probes-to use in a pod spec.+The metrics address exposes, next to the Go runtime metrics:++| Metric | Meaning |+|---|---|+| `sam_router_ready` | `1` once the router is enrolled and the host is up. |+| `sam_router_draining` | `1` from the moment the router starts stopping until the process exits. |+| `sam_router_connections{direction}` | Open connections, `inbound` and `outbound`. |+| `sam_router_connection_watermark{level}` | The `low` and `high` watermarks in force. The gap between `inbound` connections and the `high` watermark is the free capacity. |+| `sam_router_conns_per_source_ip_limit` | The per-address cap in force. |+| `sam_router_relay_limit{limit}` | The relay budget in force: `reservations`, `reservations_per_ip` and `circuits_per_peer`. |+| `sam_router_connected_peers`, `sam_router_authenticated_peers`, `sam_router_banned_peers` | Peers in each state. |+| `sam_router_dht_routing_table_size` | DHT routing table size. |+| `sam_router_auth_handshakes_total{result}`, `sam_router_lease_renewals_total{result}` | Handshakes and lease renewals by outcome. |+| `sam_router_inbound_connections_refused_total` | Inbound connections refused by the per-address cap. |+| `libp2p_relaysvc_*` | The relay service's own counters: `reservations_total{type}` and `connections_total{type}` (`opened`, `closed`, `renewed`), requests by response status, rejections and bytes relayed. The difference between `opened` and `closed` is the number in use; read it against `sam_router_relay_limit` to see how close the relay is to its budget. |++A fleet is at capacity when `sam_router_connections{direction="inbound"}`+approaches the `high` watermark or open relay reservations approach+`sam_router_relay_limit{limit="reservations"}` on every router that is not+draining; both are the inputs for an autoscaler. `/healthz` and `/readyz`+on the same address are the probes to use in a pod spec.