Skip to content

Agent Mesh

router: export capacity, draining and the relay budget on /metrics

Antonio Ojeaa577440932e9
diff --git a/internal/router/metrics.go b/internal/router/metrics.go
index 21dc5498..39f9c8d0 100644
--- a/internal/router/metrics.go
+++ b/internal/router/metrics.go
@@ -20,6 +20,7 @@ import (
"sync"
"time"
+ "github.com/libp2p/go-libp2p/core/network"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -81,8 +82,12 @@ type routerStateCollector struct {
infoDesc *prometheus.Desc
readyDesc *prometheus.Desc
+ drainingDesc *prometheus.Desc
authenticatedDesc *prometheus.Desc
connectedDesc *prometheus.Desc
+ connectionsDesc *prometheus.Desc
+ watermarkDesc *prometheus.Desc
+ relayLimitDesc *prometheus.Desc
bannedDesc *prometheus.Desc
dhtDesc *prometheus.Desc
biscuitExpiryDesc *prometheus.Desc
@@ -99,6 +104,10 @@ func newRouterStateCollector(r *Router) *routerStateCollector {
"sam_router_ready",
"1 once the router is enrolled and its libp2p host is online",
nil, nil),
+ drainingDesc: prometheus.NewDesc(
+ "sam_router_draining",
+ "1 once the router has withdrawn its lease and is shutting down",
+ nil, nil),
authenticatedDesc: prometheus.NewDesc(
"sam_router_authenticated_peers",
"Connected peers that have completed the mesh authentication handshake",
@@ -107,6 +116,18 @@ func newRouterStateCollector(r *Router) *routerStateCollector {
"sam_router_connected_peers",
"Peers with an open libp2p connection, authenticated or not",
nil, nil),
+ connectionsDesc: prometheus.NewDesc(
+ "sam_router_connections",
+ "Open libp2p connections by direction; a peer may hold several",
+ []string{"direction"}, nil),
+ watermarkDesc: prometheus.NewDesc(
+ "sam_router_connection_watermark",
+ "Connection manager limits: above high, connections are trimmed down to low",
+ []string{"level"}, nil),
+ relayLimitDesc: prometheus.NewDesc(
+ "sam_router_relay_limit",
+ "Relay budgets in force, to read libp2p_relaysvc_* against",
+ []string{"limit"}, nil),
bannedDesc: prometheus.NewDesc(
"sam_router_banned_peers",
"Peers on the ban list synced from the control plane",
@@ -124,7 +145,8 @@ func newRouterStateCollector(r *Router) *routerStateCollector {
func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
- c.infoDesc, c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc, connsPerSourceIPDesc,
+ c.infoDesc, c.readyDesc, c.drainingDesc, c.authenticatedDesc, c.connectedDesc, c.connectionsDesc, c.watermarkDesc, c.relayLimitDesc,
+ c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc, connsPerSourceIPDesc,
} {
ch <- d
}
@@ -133,6 +155,16 @@ func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {
func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {
r := c.r
ch <- prometheus.MustNewConstMetric(connsPerSourceIPDesc, prometheus.GaugeValue, float64(effectiveConnsPerSourceIP(r.config.ConnsPerSourceIP)))
+ ch <- prometheus.MustNewConstMetric(c.watermarkDesc, prometheus.GaugeValue, float64(r.config.LowWaterMark), "low")
+ ch <- prometheus.MustNewConstMetric(c.watermarkDesc, prometheus.GaugeValue, float64(r.config.HighWaterMark), "high")
+ ch <- prometheus.MustNewConstMetric(c.relayLimitDesc, prometheus.GaugeValue, float64(r.config.RelayMaxReservations), "reservations")
+ ch <- prometheus.MustNewConstMetric(c.relayLimitDesc, prometheus.GaugeValue, float64(r.config.ConnsPerSourceIP), "reservations_per_ip")
+ ch <- prometheus.MustNewConstMetric(c.relayLimitDesc, prometheus.GaugeValue, float64(r.config.RelayMaxCircuits), "circuits_per_peer")
+ draining := 0.0
+ if r.shutdown.Load() {
+ draining = 1
+ }
+ ch <- prometheus.MustNewConstMetric(c.drainingDesc, prometheus.GaugeValue, draining)
// isReady is stored after Host and DHT are assigned, so observing it
// true is what makes reading them from this goroutine safe.
if !r.isReady.Load() {
@@ -143,6 +175,16 @@ func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, r.Host.ID().String())
ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.authenticatedPeers)))
ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(r.Host.Network().Peers())))
+ var inbound, outbound int
+ for _, conn := range r.Host.Network().Conns() {
+ if conn.Stat().Direction == network.DirInbound {
+ inbound++
+ } else {
+ outbound++
+ }
+ }
+ ch <- prometheus.MustNewConstMetric(c.connectionsDesc, prometheus.GaugeValue, float64(inbound), "inbound")
+ ch <- prometheus.MustNewConstMetric(c.connectionsDesc, prometheus.GaugeValue, float64(outbound), "outbound")
ch <- prometheus.MustNewConstMetric(c.bannedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.bannedPeers)))
ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(r.DHT.RoutingTable().Size()))
diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.go
index 29877713..7db4b323 100644
--- a/internal/router/metrics_test.go
+++ b/internal/router/metrics_test.go
@@ -101,6 +101,13 @@ func TestRouterMetricsListener(t *testing.T) {
`sam_router_info{peer_id="` + host.ID().String() + `"} 1`,
// The derived cap: a quarter of the default high watermark.
"sam_router_conns_per_source_ip_limit 1000",
+ "sam_router_draining 0",
+ `sam_router_connections{direction="inbound"} 0`,
+ `sam_router_connections{direction="outbound"} 0`,
+ `sam_router_connection_watermark{level="high"} 4000`,
+ `sam_router_relay_limit{limit="reservations"} 4000`,
+ `sam_router_relay_limit{limit="reservations_per_ip"} 1000`,
+ `sam_router_relay_limit{limit="circuits_per_peer"} 1024`,
"sam_router_inbound_connections_refused_total",
"sam_router_authenticated_peers 2",
"sam_router_banned_peers 1",
diff --git a/internal/router/router.go b/internal/router/router.go
index 49ed49ef..9ea078fe 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -495,10 +495,12 @@ func (r *Router) Start() (retErr error) {
return err
}
- // Setup Relay
+ // Setup Relay. Its counters (reservations, circuits, refusals by
+ // status) go to the default registry the metrics listener serves.
_, err = relay.New(hostNode, relay.WithACL(&relayACL{r: r}),
relay.WithResources(relayResources(r.config.RelayMaxReservations, r.config.RelayMaxCircuits, r.config.ConnsPerSourceIP)),
- relay.WithLimit(relayLimit(r.config.RelayLimitDuration, r.config.RelayLimitData)))
+ relay.WithLimit(relayLimit(r.config.RelayLimitDuration, r.config.RelayLimitData)),
+ relay.WithMetricsTracer(relay.NewMetricsTracer()))
if err != nil {
return err
}
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index 62fa1e80..c7d53d67 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -76,8 +76,24 @@ important state. Nodes reconnect and publish their services again.
## Metrics
-The metrics address exposes `sam_router_connected_peers`,
-`sam_router_dht_routing_table_size`, `sam_router_auth_handshakes_total` (by
-outcome) and `sam_router_lease_renewals_total` (by outcome), next to the Go
-runtime metrics. `/healthz` and `/readyz` on the same address are the probes
-to use in a pod spec.
+The metrics address exposes, next to the Go runtime metrics:
+
+| Metric | Meaning |
+|---|---|
+| `sam_router_ready` | `1` once the router is enrolled and the host is up. |
+| `sam_router_draining` | `1` from the moment the router starts stopping until the process exits. |
+| `sam_router_connections{direction}` | Open connections, `inbound` and `outbound`. |
+| `sam_router_connection_watermark{level}` | The `low` and `high` watermarks in force. The gap between `inbound` connections and the `high` watermark is the free capacity. |
+| `sam_router_conns_per_source_ip_limit` | The per-address cap in force. |
+| `sam_router_relay_limit{limit}` | The relay budget in force: `reservations`, `reservations_per_ip` and `circuits_per_peer`. |
+| `sam_router_connected_peers`, `sam_router_authenticated_peers`, `sam_router_banned_peers` | Peers in each state. |
+| `sam_router_dht_routing_table_size` | DHT routing table size. |
+| `sam_router_auth_handshakes_total{result}`, `sam_router_lease_renewals_total{result}` | Handshakes and lease renewals by outcome. |
+| `sam_router_inbound_connections_refused_total` | Inbound connections refused by the per-address cap. |
+| `libp2p_relaysvc_*` | The relay service's own counters: `reservations_total{type}` and `connections_total{type}` (`opened`, `closed`, `renewed`), requests by response status, rejections and bytes relayed. The difference between `opened` and `closed` is the number in use; read it against `sam_router_relay_limit` to see how close the relay is to its budget. |
+
+A fleet is at capacity when `sam_router_connections{direction="inbound"}`
+approaches the `high` watermark or open relay reservations approach
+`sam_router_relay_limit{limit="reservations"}` on every router that is not
+draining; both are the inputs for an autoscaler. `/healthz` and `/readyz`
+on the same address are the probes to use in a pod spec.