Skip to content

Agent Mesh

router: serve /metrics, /healthz and /readyz on --metrics-addr

Antonio Ojea4a33506b5d65
diff --git a/cmd/sam-router/main.go b/cmd/sam-router/main.go
index ff8ca0a3..da37ad63 100644
--- a/cmd/sam-router/main.go
+++ b/cmd/sam-router/main.go
@@ -45,6 +45,7 @@ var (
dhtMaxRecordAge time.Duration
lowWaterMark int
highWaterMark int
+ metricsAddr string
)
var logger = golog.Logger("sam-router-cli")
@@ -84,6 +85,7 @@ func main() {
DHTMaxRecordAge: dhtMaxRecordAge,
LowWaterMark: lowWaterMark,
HighWaterMark: highWaterMark,
+ MetricsAddr: metricsAddr,
}
r, err := router.NewRouter(cmd.Context(), opts)
@@ -122,6 +124,7 @@ func main() {
rootCmd.Flags().DurationVar(&dhtMaxRecordAge, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")
rootCmd.Flags().IntVar(&lowWaterMark, "low-watermark", 1000, "Connection manager low watermark limit")
rootCmd.Flags().IntVar(&highWaterMark, "high-watermark", 4000, "Connection manager high watermark limit")
+ rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address (e.g. 0.0.0.0:9090); unauthenticated, off by default")
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
diff --git a/internal/router/config.go b/internal/router/config.go
index 7debe25f..58cf29cd 100644
--- a/internal/router/config.go
+++ b/internal/router/config.go
@@ -57,6 +57,10 @@ type Options struct {
// to a non-loopback host. Off by default: whoever answers that URL is
// the trust root.
AllowInsecureControlPlane bool
+ // MetricsAddr, when set, serves /metrics, /healthz and /readyz on a
+ // plain HTTP listener separate from the libp2p ports. Off by default:
+ // the listener is unauthenticated, so the operator names where it binds.
+ MetricsAddr string
}
// Default sets default values for options.
diff --git a/internal/router/metrics.go b/internal/router/metrics.go
new file mode 100644
index 00000000..ee50a481
--- /dev/null
+++ b/internal/router/metrics.go
@@ -0,0 +1,199 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "net"
+ "net/http"
+ "sync"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// Outcomes of an inbound /sam/auth handshake.
+const (
+ handshakeOK = "ok"
+ handshakeBanned = "banned"
+ handshakeRateLimited = "rate_limited"
+ handshakeReadFailed = "read_failed"
+ handshakeInvalidFrame = "invalid_frame"
+ handshakeUnauthorized = "unauthorized"
+)
+
+// Outcomes of a control-plane lease renewal attempt.
+const (
+ leaseOK = "ok"
+ leaseUnreachable = "unreachable"
+ leaseUnauthorized = "unauthorized"
+ leaseRejected = "rejected"
+)
+
+var (
+ authHandshakesTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "sam_router_auth_handshakes_total",
+ Help: "Inbound mesh authentication handshakes by outcome",
+ },
+ []string{"result"},
+ )
+
+ leaseRenewalsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "sam_router_lease_renewals_total",
+ Help: "Control-plane lease renewal attempts by outcome",
+ },
+ []string{"result"},
+ )
+)
+
+// Every outcome exists from the first scrape, so a rate() over one that has
+// not happened yet reads 0 rather than no data.
+func init() {
+ for _, v := range []string{handshakeOK, handshakeBanned, handshakeRateLimited, handshakeReadFailed, handshakeInvalidFrame, handshakeUnauthorized} {
+ authHandshakesTotal.WithLabelValues(v)
+ }
+ for _, v := range []string{leaseOK, leaseUnreachable, leaseUnauthorized, leaseRejected} {
+ leaseRenewalsTotal.WithLabelValues(v)
+ }
+}
+
+// routerStateCollector exports the live view only the router has: who has
+// proven mesh membership on this hop, as opposed to who merely holds a
+// transport connection. Nothing is emitted before Start has finished, since
+// the host and DHT do not exist yet.
+type routerStateCollector struct {
+ r *Router
+
+ readyDesc *prometheus.Desc
+ authenticatedDesc *prometheus.Desc
+ connectedDesc *prometheus.Desc
+ bannedDesc *prometheus.Desc
+ dhtDesc *prometheus.Desc
+ biscuitExpiryDesc *prometheus.Desc
+}
+
+func newRouterStateCollector(r *Router) *routerStateCollector {
+ return &routerStateCollector{
+ r: r,
+ readyDesc: prometheus.NewDesc(
+ "sam_router_ready",
+ "1 once the router is enrolled and its libp2p host is online",
+ nil, nil),
+ authenticatedDesc: prometheus.NewDesc(
+ "sam_router_authenticated_peers",
+ "Connected peers that have completed the mesh authentication handshake",
+ nil, nil),
+ connectedDesc: prometheus.NewDesc(
+ "sam_router_connected_peers",
+ "Peers with an open libp2p connection, authenticated or not",
+ nil, nil),
+ bannedDesc: prometheus.NewDesc(
+ "sam_router_banned_peers",
+ "Peers on the ban list synced from the control plane",
+ nil, nil),
+ dhtDesc: prometheus.NewDesc(
+ "sam_router_dht_routing_table_size",
+ "Peers in the Kademlia routing table",
+ nil, nil),
+ biscuitExpiryDesc: prometheus.NewDesc(
+ "sam_router_biscuit_expiry_timestamp_seconds",
+ "Unix time the router's own mesh credential expires",
+ nil, nil),
+ }
+}
+
+func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {
+ for _, d := range []*prometheus.Desc{
+ c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc,
+ } {
+ ch <- d
+ }
+}
+
+func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {
+ r := c.r
+ // isReady is stored after Host and DHT are assigned, so observing it
+ // true is what makes reading them from this goroutine safe.
+ if !r.isReady.Load() {
+ ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 0)
+ return
+ }
+ ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 1)
+ ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.authenticatedPeers)))
+ ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(r.Host.Network().Peers())))
+ ch <- prometheus.MustNewConstMetric(c.bannedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.bannedPeers)))
+ ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(r.DHT.RoutingTable().Size()))
+
+ r.keysMu.RLock()
+ expiry := r.biscuitExpiration
+ r.keysMu.RUnlock()
+ if !expiry.IsZero() {
+ ch <- prometheus.MustNewConstMetric(c.biscuitExpiryDesc, prometheus.GaugeValue, float64(expiry.Unix()))
+ }
+}
+
+func syncMapLen(m *sync.Map) int {
+ n := 0
+ m.Range(func(_, _ any) bool {
+ n++
+ return true
+ })
+ return n
+}
+
+// serveMetrics opens the operator listener: /metrics, /healthz and /readyz.
+// It is off unless an address is configured, and never shares a port with the
+// libp2p listeners, so it can stay cluster-internal while the mesh port is
+// public. libp2p's own metrics land in the default registry, so exposing it
+// here is what makes relay reservations and connection churn visible.
+func (r *Router) serveMetrics(addr string) error {
+ listener, err := net.Listen("tcp", addr)
+ if err != nil {
+ return err
+ }
+
+ reg := prometheus.NewRegistry()
+ reg.MustRegister(newRouterStateCollector(r))
+
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", promhttp.HandlerFor(
+ prometheus.Gatherers{prometheus.DefaultGatherer, reg},
+ promhttp.HandlerOpts{},
+ ))
+ mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+ mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
+ if !r.isReady.Load() {
+ http.Error(w, "router is not online yet", http.StatusServiceUnavailable)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ })
+
+ r.metricsServer = &http.Server{
+ Handler: mux,
+ ReadHeaderTimeout: 10 * time.Second,
+ }
+ r.metricsAddr = listener.Addr()
+ go func() {
+ _ = r.metricsServer.Serve(listener)
+ }()
+ logger.Infof("Router metrics listening on http://%s", r.metricsAddr)
+ return nil
+}
diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.go
new file mode 100644
index 00000000..3e17ecfd
--- /dev/null
+++ b/internal/router/metrics_test.go
@@ -0,0 +1,124 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/sam/api"
+ "github.com/libp2p/go-libp2p"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+func getBody(t *testing.T, url string) (int, string) {
+ t.Helper()
+ resp, err := http.Get(url)
+ if err != nil {
+ t.Fatalf("GET %s: %v", url, err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ return resp.StatusCode, string(body)
+}
+
+func TestRouterMetricsListener(t *testing.T) {
+ ctx := context.Background()
+ r, err := NewRouter(ctx, Options{BiscuitTimeout: time.Second, RequiredRole: api.RoleRouter})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := r.serveMetrics("127.0.0.1:0"); err != nil {
+ t.Fatal(err)
+ }
+ base := "http://" + r.metricsAddr.String()
+
+ // Before Start completes: alive, not ready, and nothing read from a host
+ // that does not exist yet.
+ if code, _ := getBody(t, base+"/healthz"); code != http.StatusOK {
+ t.Errorf("/healthz = %d before ready", code)
+ }
+ if code, _ := getBody(t, base+"/readyz"); code != http.StatusServiceUnavailable {
+ t.Errorf("/readyz = %d before ready, want 503", code)
+ }
+ code, body := getBody(t, base+"/metrics")
+ if code != http.StatusOK {
+ t.Fatalf("/metrics = %d: %s", code, body)
+ }
+ if !strings.Contains(body, "sam_router_ready 0") {
+ t.Error("/metrics missing sam_router_ready 0 before start")
+ }
+ if strings.Contains(body, "sam_router_authenticated_peers") {
+ t.Error("/metrics exported host state before the host existed")
+ }
+
+ host, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = host.Close() }()
+ kad, err := dht.New(host, dht.Mode(dht.ModeServer))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = kad.Close() }()
+
+ r.Host = host
+ r.DHT = kad
+ r.authenticatedPeers.Store(peer.ID("peer-a"), true)
+ r.authenticatedPeers.Store(peer.ID("peer-b"), true)
+ r.bannedPeers.Store(peer.ID("peer-x"), true)
+ expiry := time.Now().Add(time.Hour).Truncate(time.Second)
+ r.keysMu.Lock()
+ r.biscuitExpiration = expiry
+ r.keysMu.Unlock()
+ r.isReady.Store(true)
+
+ if code, _ := getBody(t, base+"/readyz"); code != http.StatusOK {
+ t.Errorf("/readyz = %d once ready", code)
+ }
+ _, body = getBody(t, base+"/metrics")
+ for _, want := range []string{
+ "sam_router_ready 1",
+ "sam_router_authenticated_peers 2",
+ "sam_router_banned_peers 1",
+ "sam_router_connected_peers 0",
+ "sam_router_dht_routing_table_size 0",
+ // The text format renders values with strconv 'g', so a unix time
+ // comes out in scientific notation.
+ "sam_router_biscuit_expiry_timestamp_seconds " + strconv.FormatFloat(float64(expiry.Unix()), 'g', -1, 64),
+ // libp2p registers into the default registry, which the listener
+ // serves alongside the router's own state.
+ "libp2p_",
+ "sam_router_auth_handshakes_total",
+ } {
+ if !strings.Contains(body, want) {
+ t.Errorf("/metrics missing %q", want)
+ }
+ }
+
+ if err := r.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := http.Get(base + "/healthz"); err == nil {
+ t.Error("metrics listener still answering after Close")
+ }
+}
diff --git a/internal/router/router.go b/internal/router/router.go
index 02899279..a8f4a470 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -136,6 +136,9 @@ type Router struct {
wg sync.WaitGroup
isReady atomic.Bool
shutdown bool
+
+ metricsServer *http.Server
+ metricsAddr net.Addr
}
// NewRouter initializes the router.
@@ -236,6 +239,14 @@ func perIPConnResourceManager(limit int) (network.ResourceManager, error) {
// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.
func (r *Router) Start() error {
+ // The operator listener comes up first so /healthz answers while
+ // enrollment is still in flight; /readyz turns 200 at the end.
+ if r.config.MetricsAddr != "" {
+ if err := r.serveMetrics(r.config.MetricsAddr); err != nil {
+ return fmt.Errorf("failed to start metrics listener: %w", err)
+ }
+ }
+
// 1. Load or Generate persistent identity key
priv, err := getOrGeneratePeerKey(r.config.KeysDBPath)
if err != nil {
@@ -913,6 +924,7 @@ func (r *Router) renewLease() {
resp, err := client.Post(r.config.ControlPlaneURL+"/routers/lease", "application/x-protobuf", bytes.NewReader(data))
if err != nil {
logger.Errorf("Failed to renew lease with control plane: %v", err)
+ leaseRenewalsTotal.WithLabelValues(leaseUnreachable).Inc()
return
}
@@ -921,6 +933,7 @@ func (r *Router) renewLease() {
if resp.StatusCode == http.StatusUnauthorized && attempt == 0 {
logger.Warnf("Control plane lease renewal rejected (401 Unauthorized: %s), attempting recovery...", string(body))
+ leaseRenewalsTotal.WithLabelValues(leaseUnauthorized).Inc()
if err := r.recoverAfterLease401(); err != nil {
logger.Errorf("Recovery failed after 401 Unauthorized lease renewal: %v", err)
return
@@ -931,19 +944,23 @@ func (r *Router) renewLease() {
if resp.StatusCode != http.StatusOK {
logger.Errorf("Control plane lease renewal rejected, status %s: %s", resp.Status, string(body))
+ leaseRenewalsTotal.WithLabelValues(leaseRejected).Inc()
return
}
var leaseResp api.RouterLeaseResponse
if err := proto.Unmarshal(body, &leaseResp); err != nil {
logger.Errorf("Failed to parse lease response: %v", err)
+ leaseRenewalsTotal.WithLabelValues(leaseRejected).Inc()
return
}
if !leaseResp.Success {
logger.Errorf("Lease renewal failed: %s", leaseResp.Error)
+ leaseRenewalsTotal.WithLabelValues(leaseRejected).Inc()
} else {
logger.Debugf("Lease renewed successfully. Expires at: %s", time.Unix(leaseResp.ExpiresAt, 0))
+ leaseRenewalsTotal.WithLabelValues(leaseOK).Inc()
}
return
}
@@ -1137,12 +1154,14 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
if _, banned := r.bannedPeers.Load(remotePeer); banned {
logger.Warnf("[AuthN] Rejecting authentication for banned peer %s", remotePeer)
+ authHandshakesTotal.WithLabelValues(handshakeBanned).Inc()
_ = s.Reset()
return
}
if r.handshakeLimiter != nil && !r.handshakeLimiter.Allow(remotePeer.String()) {
logger.Warnf("[AuthN] Handshake rate limit exceeded for %s", remotePeer)
+ authHandshakesTotal.WithLabelValues(handshakeRateLimited).Inc()
_ = s.Reset()
return
}
@@ -1154,6 +1173,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
msg, err := reader.ReadMsg()
if err != nil {
logger.Errorf("[AuthN] Failed to read handshake from %s: %v", remotePeer, err)
+ authHandshakesTotal.WithLabelValues(handshakeReadFailed).Inc()
return
}
defer reader.ReleaseMsg(msg)
@@ -1161,6 +1181,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
var exchange api.AuthFrame
if err := proto.Unmarshal(msg, &exchange); err != nil {
logger.Warnf("[AuthN] Invalid protobuf from %s", remotePeer)
+ authHandshakesTotal.WithLabelValues(handshakeInvalidFrame).Inc()
return
}
@@ -1168,11 +1189,13 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
_, err = identity.VerifyBiscuit(exchange.Biscuit, remotePeer, r.getTrustedPublicKeys(), r.config.BiscuitTimeout)
if err != nil {
logger.Warnf("[AuthN] Authorization failed for peer %s: %v", remotePeer, err)
+ authHandshakesTotal.WithLabelValues(handshakeUnauthorized).Inc()
_ = s.Reset()
return
}
r.authenticatedPeers.Store(remotePeer, true)
+ authHandshakesTotal.WithLabelValues(handshakeOK).Inc()
logger.Infof("[AuthN] Successfully authenticated peer %s", remotePeer)
// Send mutual response (our biscuit)
@@ -1250,6 +1273,11 @@ func (r *Router) Close() error {
r.cancel()
var errs []error
+ if r.metricsServer != nil {
+ if err := r.metricsServer.Close(); err != nil {
+ errs = append(errs, err)
+ }
+ }
if r.DHT != nil {
if err := r.DHT.Close(); err != nil {
errs = append(errs, err)