Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: export mesh-membership gauges and add --metrics-addr
Antonio Ojea06494ec42286
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.goindex 318e3cdf..9bed6b41 100644--- a/cmd/sam-node/main.go+++ b/cmd/sam-node/main.go@@ -62,6 +62,7 @@ var (clientSecretFlag stringcontrolPlanePublicKeyFlag stringbindAddrFlag string+ metricsAddrFlag stringsocketPathFlag stringmeshFlag stringdiscoveryIntervalFlag string@@ -644,6 +645,14 @@ func main() {}}()+ if metricsAddrFlag != "" {+ metricsSrv, err := node.StartMetricsServer(meshNode, metricsAddrFlag)+ if err != nil {+ logger.Fatalf("Failed to start metrics server: %v", err)+ }+ defer func() { _ = metricsSrv.Close() }()+ }+fmt.Printf("SAM Node Online.\nPeerID: %s\nListening on: %v\n", meshNode.Host.ID(), meshNode.Host.Addrs())// Block forever@@ -836,6 +845,7 @@ func main() {runCmd.Flags().StringVar(&controlPlanePublicKeyFlag, "control-plane-public-key", "", "Control plane public key (32-byte Hex)")runCmd.Flags().StringVar(&bindAddrFlag, "bind-addr", "127.0.0.1:8080", "Local TCP address for the HTTP server (MCP and Sidecar API); pass an empty value to serve only on the Unix socket")runCmd.Flags().StringVar(&socketPathFlag, "socket-path", "", "Unix socket serving the same API, where the socket's owner-only permissions replace the API token (defaults to <data-dir>/"+node.DefaultSocketName+"; pass an empty value to disable)")+ runCmd.Flags().StringVar(&metricsAddrFlag, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address without authentication (e.g. 0.0.0.0:9090), for scrapers and probes that hold no API token; off by default")runCmd.Flags().StringVar(&meshFlag, "mesh", node.DefaultMeshName, "Mesh federation name")runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", node.DefaultDiscoveryInterval, "Polling interval for DHT discovery")runCmd.Flags().DurationVar(&monitorBootstrapFlag, "monitor-bootstrap", 2*time.Minute, "Initial wait before monitoring router connection")diff --git a/internal/node/node.go b/internal/node/node.goindex af2d4ae8..b80e2ae9 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -64,6 +64,7 @@ import ("github.com/libp2p/go-msgio""github.com/multiformats/go-multiaddr"madns "github.com/multiformats/go-multiaddr-dns"+ "github.com/prometheus/client_golang/prometheus""google.golang.org/protobuf/proto")@@ -178,6 +179,10 @@ type SamNode struct {BiscuitTimeout time.DurationcachedIdentity atomic.Valuelogger *golog.ZapEventLogger++ // metricsRegistry holds this node's state collector; see metricsHandler.+ metricsOnce sync.Once+ metricsRegistry *prometheus.Registry}// UpdateRelays updates the current relays used by AutoRelay.diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.gonew file mode 100644index 00000000..0519d259--- /dev/null+++ b/internal/node/node_metrics.go@@ -0,0 +1,182 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "errors"+ "net"+ "net/http"+ "strings"+ "time"++ "github.com/google/sam/api"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promhttp"+)++// nodeStateCollector exports the node's own view of its mesh membership: the+// same figures GET /debug/mesh-info and /debug/token-info answer on demand,+// as gauges a scraper can keep. A node that runs unattended, such as a+// canary, becomes a continuous probe of the mesh this way.+type nodeStateCollector struct {+ n *SamNode++ readyDesc *prometheus.Desc+ meshConnectedDesc *prometheus.Desc+ connectedDesc *prometheus.Desc+ authenticatedDesc *prometheus.Desc+ dhtDesc *prometheus.Desc+ biscuitExpiryDesc *prometheus.Desc+ servicesDesc *prometheus.Desc+}++func newNodeStateCollector(n *SamNode) *nodeStateCollector {+ return &nodeStateCollector{+ n: n,+ readyDesc: prometheus.NewDesc(+ "sam_node_ready",+ "1 once the libp2p host, DHT and identity store exist",+ nil, nil),+ meshConnectedDesc: prometheus.NewDesc(+ "sam_node_mesh_connected",+ "1 while the node holds an authenticated connection to a router",+ nil, nil),+ connectedDesc: prometheus.NewDesc(+ "sam_node_connected_peers",+ "Peers with an open libp2p connection, authenticated or not",+ nil, nil),+ authenticatedDesc: prometheus.NewDesc(+ "sam_node_authenticated_peers",+ "Peers that have completed the mesh authentication handshake with this node",+ nil, nil),+ dhtDesc: prometheus.NewDesc(+ "sam_node_dht_routing_table_size",+ "Peers in the Kademlia routing table",+ nil, nil),+ biscuitExpiryDesc: prometheus.NewDesc(+ "sam_node_biscuit_expiry_timestamp_seconds",+ "Unix time the node's mesh credential expires",+ nil, nil),+ servicesDesc: prometheus.NewDesc(+ "sam_node_services_registered",+ "Local services this node offers to the mesh, by type",+ []string{"type"}, nil),+ }+}++func (c *nodeStateCollector) Describe(ch chan<- *prometheus.Desc) {+ for _, d := range []*prometheus.Desc{+ c.readyDesc, c.meshConnectedDesc, c.connectedDesc, c.authenticatedDesc,+ c.dhtDesc, c.biscuitExpiryDesc, c.servicesDesc,+ } {+ ch <- d+ }+}++func (c *nodeStateCollector) Collect(ch chan<- prometheus.Metric) {+ n := c.n+ if n.debugReady() != nil {+ ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 0)+ return+ }+ ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 1)++ connected := 0.0+ if n.IsConnected() {+ connected = 1+ }+ ch <- prometheus.MustNewConstMetric(c.meshConnectedDesc, prometheus.GaugeValue, connected)+ ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(n.Host.Network().Peers())))++ authenticated := 0+ n.authPeers.Range(func(_, _ any) bool {+ authenticated+++ return true+ })+ ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(authenticated))+ ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(n.DHT.RoutingTable().Size()))++ if exp, err := n.Store.LoadIdentityExpiration(); err == nil && exp > 0 {+ ch <- prometheus.MustNewConstMetric(c.biscuitExpiryDesc, prometheus.GaugeValue, float64(exp))+ }++ if n.services != nil {+ byType := map[string]int{}+ for _, svc := range n.services.List(api.ServiceType_SERVICE_TYPE_UNSPECIFIED) {+ byType[serviceTypeLabel(svc.Type)]+++ }+ for t, count := range byType {+ ch <- prometheus.MustNewConstMetric(c.servicesDesc, prometheus.GaugeValue, float64(count), t)+ }+ }+}++// serviceTypeLabel turns SERVICE_TYPE_MCP into "mcp".+func serviceTypeLabel(t api.ServiceType) string {+ return strings.ToLower(strings.TrimPrefix(t.String(), "SERVICE_TYPE_"))+}++// metricsHandler serves the process-wide registry, which carries the request+// and inference counters and the Go runtime, alongside this node's own state.+// The state collector is per node so two nodes in one process (tests,+// sam-one) never fight over a registration.+func (n *SamNode) metricsHandler() http.Handler {+ n.metricsOnce.Do(func() {+ n.metricsRegistry = prometheus.NewRegistry()+ n.metricsRegistry.MustRegister(newNodeStateCollector(n))+ })+ return promhttp.HandlerFor(+ prometheus.Gatherers{prometheus.DefaultGatherer, n.metricsRegistry},+ promhttp.HandlerOpts{},+ )+}++// StartMetricsServer serves /metrics, /healthz and /readyz on addr with no+// authentication, for a scraper and a kubelet that hold no sidecar token.+// The sidecar's own /metrics stays token-gated: this listener exists so a+// socket-only node, which has no TCP port at all, can still be observed. It+// is off unless an operator names the address, since nothing on it is gated.+func StartMetricsServer(node *SamNode, addr string) (*http.Server, error) {+ if addr == "" {+ return nil, errors.New("no metrics address configured")+ }+ listener, err := net.Listen("tcp", addr)+ if err != nil {+ return nil, err+ }++ mux := http.NewServeMux()+ mux.Handle("/metrics", node.metricsHandler())+ mux.HandleFunc("/healthz", handleHealthz)+ mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {+ if node.debugReady() != nil || !node.IsConnected() {+ http.Error(w, "not connected to the mesh", http.StatusServiceUnavailable)+ return+ }+ w.WriteHeader(http.StatusOK)+ })++ server := &http.Server{+ // Informational once Serve has the listener; lets callers find the port.+ Addr: listener.Addr().String(),+ Handler: mux,+ ReadHeaderTimeout: 10 * time.Second,+ }+ go func() {+ _ = server.Serve(listener)+ }()+ logger.Infof("Metrics listening on http://%s", listener.Addr())+ return server, nil+}diff --git a/internal/node/node_metrics_test.go b/internal/node/node_metrics_test.gonew file mode 100644index 00000000..5747f9ed--- /dev/null+++ b/internal/node/node_metrics_test.go@@ -0,0 +1,150 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "io"+ "net/http"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/libp2p/go-libp2p"+ dht "github.com/libp2p/go-libp2p-kad-dht"+ "github.com/libp2p/go-libp2p/core/peer"+)++func getMetricsBody(t *testing.T, url string) (int, string) {+ t.Helper()+ resp, err := http.Get(url)+ if err != nil {+ t.Fatalf("GET %s: %v", url, err)+ }+ defer func() { _ = resp.Body.Close() }()+ body, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, string(body)+}++func TestNodeMetricsServer(t *testing.T) {+ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = store.Close() }()++ // Half-built, as a node is between NewSamNode and Start: alive, not+ // ready, and nothing read from a host that does not exist yet.+ node := &SamNode{Store: store, services: NewServiceRegistry(&fakeDHT{}, 0)}+ srv, err := StartMetricsServer(node, "127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = srv.Close() }()+ base := "http://" + srv.Addr+ if base == "http://" {+ t.Fatal("metrics server did not record its address")+ }++ if code, _ := getMetricsBody(t, base+"/healthz"); code != http.StatusOK {+ t.Errorf("/healthz = %d before ready", code)+ }+ if code, _ := getMetricsBody(t, base+"/readyz"); code != http.StatusServiceUnavailable {+ t.Errorf("/readyz = %d before ready, want 503", code)+ }+ code, body := getMetricsBody(t, base+"/metrics")+ if code != http.StatusOK {+ t.Fatalf("/metrics = %d: %s", code, body)+ }+ if !strings.Contains(body, "sam_node_ready 0") {+ t.Error("/metrics missing sam_node_ready 0 before start")+ }+ if strings.Contains(body, "sam_node_connected_peers") {+ t.Error("/metrics exported host state before the host existed")+ }++ host, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = host.Close() }()+ kad, err := dht.New(host, dht.Mode(dht.ModeServer))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = kad.Close() }()++ expiry := time.Now().Add(time.Hour).Unix()+ if err := store.SaveIdentityExpiration(expiry); err != nil {+ t.Fatal(err)+ }+ node.Host = host+ node.DHT = kad+ node.authPeers.Store(peer.ID("peer-a"), time.Now().Add(time.Hour))+ node.authPeers.Store(peer.ID("peer-b"), time.Now().Add(time.Hour))++ // Ready, but no authenticated router yet: the kubelet must not route to it.+ if code, _ := getMetricsBody(t, base+"/readyz"); code != http.StatusServiceUnavailable {+ t.Errorf("/readyz = %d with no router, want 503", code)+ }+ _, body = getMetricsBody(t, base+"/metrics")+ for _, want := range []string{+ "sam_node_ready 1",+ "sam_node_mesh_connected 0",+ "sam_node_connected_peers 0",+ "sam_node_authenticated_peers 2",+ "sam_node_dht_routing_table_size 0",+ "sam_node_biscuit_expiry_timestamp_seconds " + strconv.FormatFloat(float64(expiry), 'g', -1, 64),+ // The process-wide registry rides along: sidecar counters and libp2p.+ "sam_node_requests_in_flight",+ "libp2p_",+ } {+ if !strings.Contains(body, want) {+ t.Errorf("/metrics missing %q", want)+ }+ }+ if strings.Contains(body, "sam_node_services_registered") {+ t.Error("services gauge exported with an empty registry")+ }++ // An authenticated router that is actually connected flips both.+ routerHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = routerHost.Close() }()+ if err := host.Connect(t.Context(), peer.AddrInfo{ID: routerHost.ID(), Addrs: routerHost.Addrs()}); err != nil {+ t.Fatal(err)+ }+ node.mu.Lock()+ node.authenticatedRouters = map[peer.ID]bool{routerHost.ID(): true}+ node.mu.Unlock()++ if code, _ := getMetricsBody(t, base+"/readyz"); code != http.StatusOK {+ t.Errorf("/readyz = %d once a router is authenticated", code)+ }+ _, body = getMetricsBody(t, base+"/metrics")+ for _, want := range []string{"sam_node_mesh_connected 1", "sam_node_connected_peers 1"} {+ if !strings.Contains(body, want) {+ t.Errorf("/metrics missing %q", want)+ }+ }+}++func TestStartMetricsServerRequiresAddress(t *testing.T) {+ if _, err := StartMetricsServer(&SamNode{}, ""); err == nil {+ t.Fatal("expected an error for an empty address")+ }+}diff --git a/internal/node/sidecar.go b/internal/node/sidecar.goindex ff3c5cac..ea6ca1e1 100644--- a/internal/node/sidecar.go+++ b/internal/node/sidecar.go@@ -36,7 +36,6 @@ import (libp2phttp "github.com/libp2p/go-libp2p-http""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"- "github.com/prometheus/client_golang/prometheus/promhttp")// StartSidecarServer serves the node's local API on a TCP address, on a Unix@@ -51,7 +50,7 @@ func StartSidecarServer(node *SamNode, addr, socketPath, token, certFile, keyFil// Gated like the rest: the labels carry peer IDs and per-peer request counts,// and this mux is reachable by any local process over TCP. Socket callers are// unaffected, which is how every scrape in this repo reads it.- mux.Handle("/metrics", withAuth(token, true, promhttp.Handler()))+ mux.Handle("/metrics", withAuth(token, true, node.metricsHandler()))// Protected endpoints. allowAuthorizationFallback=true is safe here: none of// these ever forward the inbound Authorization header to another service.