Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: serve /metrics, /healthz and /readyz on --metrics-addr
Antonio Ojea4a33506b5d65
diff --git a/cmd/sam-router/main.go b/cmd/sam-router/main.goindex ff8ca0a3..da37ad63 100644--- a/cmd/sam-router/main.go+++ b/cmd/sam-router/main.go@@ -45,6 +45,7 @@ var (dhtMaxRecordAge time.DurationlowWaterMark inthighWaterMark int+ metricsAddr string)var logger = golog.Logger("sam-router-cli")@@ -84,6 +85,7 @@ func main() {DHTMaxRecordAge: dhtMaxRecordAge,LowWaterMark: lowWaterMark,HighWaterMark: highWaterMark,+ MetricsAddr: metricsAddr,}r, err := router.NewRouter(cmd.Context(), opts)@@ -122,6 +124,7 @@ func main() {rootCmd.Flags().DurationVar(&dhtMaxRecordAge, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")rootCmd.Flags().IntVar(&lowWaterMark, "low-watermark", 1000, "Connection manager low watermark limit")rootCmd.Flags().IntVar(&highWaterMark, "high-watermark", 4000, "Connection manager high watermark limit")+ rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address (e.g. 0.0.0.0:9090); unauthenticated, off by default")ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)defer cancel()diff --git a/internal/router/config.go b/internal/router/config.goindex 7debe25f..58cf29cd 100644--- a/internal/router/config.go+++ b/internal/router/config.go@@ -57,6 +57,10 @@ type Options struct {// to a non-loopback host. Off by default: whoever answers that URL is// the trust root.AllowInsecureControlPlane bool+ // MetricsAddr, when set, serves /metrics, /healthz and /readyz on a+ // plain HTTP listener separate from the libp2p ports. Off by default:+ // the listener is unauthenticated, so the operator names where it binds.+ MetricsAddr string}// Default sets default values for options.diff --git a/internal/router/metrics.go b/internal/router/metrics.gonew file mode 100644index 00000000..ee50a481--- /dev/null+++ b/internal/router/metrics.go@@ -0,0 +1,199 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "net"+ "net/http"+ "sync"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+ "github.com/prometheus/client_golang/prometheus/promhttp"+)++// Outcomes of an inbound /sam/auth handshake.+const (+ handshakeOK = "ok"+ handshakeBanned = "banned"+ handshakeRateLimited = "rate_limited"+ handshakeReadFailed = "read_failed"+ handshakeInvalidFrame = "invalid_frame"+ handshakeUnauthorized = "unauthorized"+)++// Outcomes of a control-plane lease renewal attempt.+const (+ leaseOK = "ok"+ leaseUnreachable = "unreachable"+ leaseUnauthorized = "unauthorized"+ leaseRejected = "rejected"+)++var (+ authHandshakesTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "sam_router_auth_handshakes_total",+ Help: "Inbound mesh authentication handshakes by outcome",+ },+ []string{"result"},+ )++ leaseRenewalsTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "sam_router_lease_renewals_total",+ Help: "Control-plane lease renewal attempts by outcome",+ },+ []string{"result"},+ )+)++// Every outcome exists from the first scrape, so a rate() over one that has+// not happened yet reads 0 rather than no data.+func init() {+ for _, v := range []string{handshakeOK, handshakeBanned, handshakeRateLimited, handshakeReadFailed, handshakeInvalidFrame, handshakeUnauthorized} {+ authHandshakesTotal.WithLabelValues(v)+ }+ for _, v := range []string{leaseOK, leaseUnreachable, leaseUnauthorized, leaseRejected} {+ leaseRenewalsTotal.WithLabelValues(v)+ }+}++// routerStateCollector exports the live view only the router has: who has+// proven mesh membership on this hop, as opposed to who merely holds a+// transport connection. Nothing is emitted before Start has finished, since+// the host and DHT do not exist yet.+type routerStateCollector struct {+ r *Router++ readyDesc *prometheus.Desc+ authenticatedDesc *prometheus.Desc+ connectedDesc *prometheus.Desc+ bannedDesc *prometheus.Desc+ dhtDesc *prometheus.Desc+ biscuitExpiryDesc *prometheus.Desc+}++func newRouterStateCollector(r *Router) *routerStateCollector {+ return &routerStateCollector{+ r: r,+ readyDesc: prometheus.NewDesc(+ "sam_router_ready",+ "1 once the router is enrolled and its libp2p host is online",+ nil, nil),+ authenticatedDesc: prometheus.NewDesc(+ "sam_router_authenticated_peers",+ "Connected peers that have completed the mesh authentication handshake",+ nil, nil),+ connectedDesc: prometheus.NewDesc(+ "sam_router_connected_peers",+ "Peers with an open libp2p connection, authenticated or not",+ nil, nil),+ bannedDesc: prometheus.NewDesc(+ "sam_router_banned_peers",+ "Peers on the ban list synced from the control plane",+ nil, nil),+ dhtDesc: prometheus.NewDesc(+ "sam_router_dht_routing_table_size",+ "Peers in the Kademlia routing table",+ nil, nil),+ biscuitExpiryDesc: prometheus.NewDesc(+ "sam_router_biscuit_expiry_timestamp_seconds",+ "Unix time the router's own mesh credential expires",+ nil, nil),+ }+}++func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {+ for _, d := range []*prometheus.Desc{+ c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc,+ } {+ ch <- d+ }+}++func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {+ r := c.r+ // isReady is stored after Host and DHT are assigned, so observing it+ // true is what makes reading them from this goroutine safe.+ if !r.isReady.Load() {+ ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 0)+ return+ }+ ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 1)+ ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.authenticatedPeers)))+ ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(r.Host.Network().Peers())))+ ch <- prometheus.MustNewConstMetric(c.bannedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.bannedPeers)))+ ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(r.DHT.RoutingTable().Size()))++ r.keysMu.RLock()+ expiry := r.biscuitExpiration+ r.keysMu.RUnlock()+ if !expiry.IsZero() {+ ch <- prometheus.MustNewConstMetric(c.biscuitExpiryDesc, prometheus.GaugeValue, float64(expiry.Unix()))+ }+}++func syncMapLen(m *sync.Map) int {+ n := 0+ m.Range(func(_, _ any) bool {+ n+++ return true+ })+ return n+}++// serveMetrics opens the operator listener: /metrics, /healthz and /readyz.+// It is off unless an address is configured, and never shares a port with the+// libp2p listeners, so it can stay cluster-internal while the mesh port is+// public. libp2p's own metrics land in the default registry, so exposing it+// here is what makes relay reservations and connection churn visible.+func (r *Router) serveMetrics(addr string) error {+ listener, err := net.Listen("tcp", addr)+ if err != nil {+ return err+ }++ reg := prometheus.NewRegistry()+ reg.MustRegister(newRouterStateCollector(r))++ mux := http.NewServeMux()+ mux.Handle("/metrics", promhttp.HandlerFor(+ prometheus.Gatherers{prometheus.DefaultGatherer, reg},+ promhttp.HandlerOpts{},+ ))+ mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusOK)+ })+ mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {+ if !r.isReady.Load() {+ http.Error(w, "router is not online yet", http.StatusServiceUnavailable)+ return+ }+ w.WriteHeader(http.StatusOK)+ })++ r.metricsServer = &http.Server{+ Handler: mux,+ ReadHeaderTimeout: 10 * time.Second,+ }+ r.metricsAddr = listener.Addr()+ go func() {+ _ = r.metricsServer.Serve(listener)+ }()+ logger.Infof("Router metrics listening on http://%s", r.metricsAddr)+ return nil+}diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.gonew file mode 100644index 00000000..3e17ecfd--- /dev/null+++ b/internal/router/metrics_test.go@@ -0,0 +1,124 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "context"+ "io"+ "net/http"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/google/sam/api"+ "github.com/libp2p/go-libp2p"+ dht "github.com/libp2p/go-libp2p-kad-dht"+ "github.com/libp2p/go-libp2p/core/peer"+)++func getBody(t *testing.T, url string) (int, string) {+ t.Helper()+ resp, err := http.Get(url)+ if err != nil {+ t.Fatalf("GET %s: %v", url, err)+ }+ defer func() { _ = resp.Body.Close() }()+ body, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, string(body)+}++func TestRouterMetricsListener(t *testing.T) {+ ctx := context.Background()+ r, err := NewRouter(ctx, Options{BiscuitTimeout: time.Second, RequiredRole: api.RoleRouter})+ if err != nil {+ t.Fatal(err)+ }+ if err := r.serveMetrics("127.0.0.1:0"); err != nil {+ t.Fatal(err)+ }+ base := "http://" + r.metricsAddr.String()++ // Before Start completes: alive, not ready, and nothing read from a host+ // that does not exist yet.+ if code, _ := getBody(t, base+"/healthz"); code != http.StatusOK {+ t.Errorf("/healthz = %d before ready", code)+ }+ if code, _ := getBody(t, base+"/readyz"); code != http.StatusServiceUnavailable {+ t.Errorf("/readyz = %d before ready, want 503", code)+ }+ code, body := getBody(t, base+"/metrics")+ if code != http.StatusOK {+ t.Fatalf("/metrics = %d: %s", code, body)+ }+ if !strings.Contains(body, "sam_router_ready 0") {+ t.Error("/metrics missing sam_router_ready 0 before start")+ }+ if strings.Contains(body, "sam_router_authenticated_peers") {+ t.Error("/metrics exported host state before the host existed")+ }++ host, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = host.Close() }()+ kad, err := dht.New(host, dht.Mode(dht.ModeServer))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = kad.Close() }()++ r.Host = host+ r.DHT = kad+ r.authenticatedPeers.Store(peer.ID("peer-a"), true)+ r.authenticatedPeers.Store(peer.ID("peer-b"), true)+ r.bannedPeers.Store(peer.ID("peer-x"), true)+ expiry := time.Now().Add(time.Hour).Truncate(time.Second)+ r.keysMu.Lock()+ r.biscuitExpiration = expiry+ r.keysMu.Unlock()+ r.isReady.Store(true)++ if code, _ := getBody(t, base+"/readyz"); code != http.StatusOK {+ t.Errorf("/readyz = %d once ready", code)+ }+ _, body = getBody(t, base+"/metrics")+ for _, want := range []string{+ "sam_router_ready 1",+ "sam_router_authenticated_peers 2",+ "sam_router_banned_peers 1",+ "sam_router_connected_peers 0",+ "sam_router_dht_routing_table_size 0",+ // The text format renders values with strconv 'g', so a unix time+ // comes out in scientific notation.+ "sam_router_biscuit_expiry_timestamp_seconds " + strconv.FormatFloat(float64(expiry.Unix()), 'g', -1, 64),+ // libp2p registers into the default registry, which the listener+ // serves alongside the router's own state.+ "libp2p_",+ "sam_router_auth_handshakes_total",+ } {+ if !strings.Contains(body, want) {+ t.Errorf("/metrics missing %q", want)+ }+ }++ if err := r.Close(); err != nil {+ t.Fatal(err)+ }+ if _, err := http.Get(base + "/healthz"); err == nil {+ t.Error("metrics listener still answering after Close")+ }+}diff --git a/internal/router/router.go b/internal/router/router.goindex 02899279..a8f4a470 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -136,6 +136,9 @@ type Router struct {wg sync.WaitGroupisReady atomic.Boolshutdown bool++ metricsServer *http.Server+ metricsAddr net.Addr}// NewRouter initializes the router.@@ -236,6 +239,14 @@ func perIPConnResourceManager(limit int) (network.ResourceManager, error) {// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.func (r *Router) Start() error {+ // The operator listener comes up first so /healthz answers while+ // enrollment is still in flight; /readyz turns 200 at the end.+ if r.config.MetricsAddr != "" {+ if err := r.serveMetrics(r.config.MetricsAddr); err != nil {+ return fmt.Errorf("failed to start metrics listener: %w", err)+ }+ }+// 1. Load or Generate persistent identity keypriv, err := getOrGeneratePeerKey(r.config.KeysDBPath)if err != nil {@@ -913,6 +924,7 @@ func (r *Router) renewLease() {resp, err := client.Post(r.config.ControlPlaneURL+"/routers/lease", "application/x-protobuf", bytes.NewReader(data))if err != nil {logger.Errorf("Failed to renew lease with control plane: %v", err)+ leaseRenewalsTotal.WithLabelValues(leaseUnreachable).Inc()return}@@ -921,6 +933,7 @@ func (r *Router) renewLease() {if resp.StatusCode == http.StatusUnauthorized && attempt == 0 {logger.Warnf("Control plane lease renewal rejected (401 Unauthorized: %s), attempting recovery...", string(body))+ leaseRenewalsTotal.WithLabelValues(leaseUnauthorized).Inc()if err := r.recoverAfterLease401(); err != nil {logger.Errorf("Recovery failed after 401 Unauthorized lease renewal: %v", err)return@@ -931,19 +944,23 @@ func (r *Router) renewLease() {if resp.StatusCode != http.StatusOK {logger.Errorf("Control plane lease renewal rejected, status %s: %s", resp.Status, string(body))+ leaseRenewalsTotal.WithLabelValues(leaseRejected).Inc()return}var leaseResp api.RouterLeaseResponseif err := proto.Unmarshal(body, &leaseResp); err != nil {logger.Errorf("Failed to parse lease response: %v", err)+ leaseRenewalsTotal.WithLabelValues(leaseRejected).Inc()return}if !leaseResp.Success {logger.Errorf("Lease renewal failed: %s", leaseResp.Error)+ leaseRenewalsTotal.WithLabelValues(leaseRejected).Inc()} else {logger.Debugf("Lease renewed successfully. Expires at: %s", time.Unix(leaseResp.ExpiresAt, 0))+ leaseRenewalsTotal.WithLabelValues(leaseOK).Inc()}return}@@ -1137,12 +1154,14 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {if _, banned := r.bannedPeers.Load(remotePeer); banned {logger.Warnf("[AuthN] Rejecting authentication for banned peer %s", remotePeer)+ authHandshakesTotal.WithLabelValues(handshakeBanned).Inc()_ = s.Reset()return}if r.handshakeLimiter != nil && !r.handshakeLimiter.Allow(remotePeer.String()) {logger.Warnf("[AuthN] Handshake rate limit exceeded for %s", remotePeer)+ authHandshakesTotal.WithLabelValues(handshakeRateLimited).Inc()_ = s.Reset()return}@@ -1154,6 +1173,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {msg, err := reader.ReadMsg()if err != nil {logger.Errorf("[AuthN] Failed to read handshake from %s: %v", remotePeer, err)+ authHandshakesTotal.WithLabelValues(handshakeReadFailed).Inc()return}defer reader.ReleaseMsg(msg)@@ -1161,6 +1181,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {var exchange api.AuthFrameif err := proto.Unmarshal(msg, &exchange); err != nil {logger.Warnf("[AuthN] Invalid protobuf from %s", remotePeer)+ authHandshakesTotal.WithLabelValues(handshakeInvalidFrame).Inc()return}@@ -1168,11 +1189,13 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {_, err = identity.VerifyBiscuit(exchange.Biscuit, remotePeer, r.getTrustedPublicKeys(), r.config.BiscuitTimeout)if err != nil {logger.Warnf("[AuthN] Authorization failed for peer %s: %v", remotePeer, err)+ authHandshakesTotal.WithLabelValues(handshakeUnauthorized).Inc()_ = s.Reset()return}r.authenticatedPeers.Store(remotePeer, true)+ authHandshakesTotal.WithLabelValues(handshakeOK).Inc()logger.Infof("[AuthN] Successfully authenticated peer %s", remotePeer)// Send mutual response (our biscuit)@@ -1250,6 +1273,11 @@ func (r *Router) Close() error {r.cancel()var errs []error+ if r.metricsServer != nil {+ if err := r.metricsServer.Close(); err != nil {+ errs = append(errs, err)+ }+ }if r.DHT != nil {if err := r.DHT.Close(); err != nil {errs = append(errs, err)