Skip to content

Agent Mesh

node: export mesh-membership gauges and add --metrics-addr

Antonio Ojea06494ec42286
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 318e3cdf..9bed6b41 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -62,6 +62,7 @@ var (
clientSecretFlag string
controlPlanePublicKeyFlag string
bindAddrFlag string
+ metricsAddrFlag string
socketPathFlag string
meshFlag string
discoveryIntervalFlag string
@@ -644,6 +645,14 @@ func main() {
}
}()
+ if metricsAddrFlag != "" {
+ metricsSrv, err := node.StartMetricsServer(meshNode, metricsAddrFlag)
+ if err != nil {
+ logger.Fatalf("Failed to start metrics server: %v", err)
+ }
+ defer func() { _ = metricsSrv.Close() }()
+ }
+
fmt.Printf("SAM Node Online.\nPeerID: %s\nListening on: %v\n", meshNode.Host.ID(), meshNode.Host.Addrs())
// Block forever
@@ -836,6 +845,7 @@ func main() {
runCmd.Flags().StringVar(&controlPlanePublicKeyFlag, "control-plane-public-key", "", "Control plane public key (32-byte Hex)")
runCmd.Flags().StringVar(&bindAddrFlag, "bind-addr", "127.0.0.1:8080", "Local TCP address for the HTTP server (MCP and Sidecar API); pass an empty value to serve only on the Unix socket")
runCmd.Flags().StringVar(&socketPathFlag, "socket-path", "", "Unix socket serving the same API, where the socket's owner-only permissions replace the API token (defaults to <data-dir>/"+node.DefaultSocketName+"; pass an empty value to disable)")
+ runCmd.Flags().StringVar(&metricsAddrFlag, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address without authentication (e.g. 0.0.0.0:9090), for scrapers and probes that hold no API token; off by default")
runCmd.Flags().StringVar(&meshFlag, "mesh", node.DefaultMeshName, "Mesh federation name")
runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", node.DefaultDiscoveryInterval, "Polling interval for DHT discovery")
runCmd.Flags().DurationVar(&monitorBootstrapFlag, "monitor-bootstrap", 2*time.Minute, "Initial wait before monitoring router connection")
diff --git a/internal/node/node.go b/internal/node/node.go
index af2d4ae8..b80e2ae9 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -64,6 +64,7 @@ import (
"github.com/libp2p/go-msgio"
"github.com/multiformats/go-multiaddr"
madns "github.com/multiformats/go-multiaddr-dns"
+ "github.com/prometheus/client_golang/prometheus"
"google.golang.org/protobuf/proto"
)
@@ -178,6 +179,10 @@ type SamNode struct {
BiscuitTimeout time.Duration
cachedIdentity atomic.Value
logger *golog.ZapEventLogger
+
+ // metricsRegistry holds this node's state collector; see metricsHandler.
+ metricsOnce sync.Once
+ metricsRegistry *prometheus.Registry
}
// UpdateRelays updates the current relays used by AutoRelay.
diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.go
new file mode 100644
index 00000000..0519d259
--- /dev/null
+++ b/internal/node/node_metrics.go
@@ -0,0 +1,182 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "errors"
+ "net"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/google/sam/api"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// nodeStateCollector exports the node's own view of its mesh membership: the
+// same figures GET /debug/mesh-info and /debug/token-info answer on demand,
+// as gauges a scraper can keep. A node that runs unattended, such as a
+// canary, becomes a continuous probe of the mesh this way.
+type nodeStateCollector struct {
+ n *SamNode
+
+ readyDesc *prometheus.Desc
+ meshConnectedDesc *prometheus.Desc
+ connectedDesc *prometheus.Desc
+ authenticatedDesc *prometheus.Desc
+ dhtDesc *prometheus.Desc
+ biscuitExpiryDesc *prometheus.Desc
+ servicesDesc *prometheus.Desc
+}
+
+func newNodeStateCollector(n *SamNode) *nodeStateCollector {
+ return &nodeStateCollector{
+ n: n,
+ readyDesc: prometheus.NewDesc(
+ "sam_node_ready",
+ "1 once the libp2p host, DHT and identity store exist",
+ nil, nil),
+ meshConnectedDesc: prometheus.NewDesc(
+ "sam_node_mesh_connected",
+ "1 while the node holds an authenticated connection to a router",
+ nil, nil),
+ connectedDesc: prometheus.NewDesc(
+ "sam_node_connected_peers",
+ "Peers with an open libp2p connection, authenticated or not",
+ nil, nil),
+ authenticatedDesc: prometheus.NewDesc(
+ "sam_node_authenticated_peers",
+ "Peers that have completed the mesh authentication handshake with this node",
+ nil, nil),
+ dhtDesc: prometheus.NewDesc(
+ "sam_node_dht_routing_table_size",
+ "Peers in the Kademlia routing table",
+ nil, nil),
+ biscuitExpiryDesc: prometheus.NewDesc(
+ "sam_node_biscuit_expiry_timestamp_seconds",
+ "Unix time the node's mesh credential expires",
+ nil, nil),
+ servicesDesc: prometheus.NewDesc(
+ "sam_node_services_registered",
+ "Local services this node offers to the mesh, by type",
+ []string{"type"}, nil),
+ }
+}
+
+func (c *nodeStateCollector) Describe(ch chan<- *prometheus.Desc) {
+ for _, d := range []*prometheus.Desc{
+ c.readyDesc, c.meshConnectedDesc, c.connectedDesc, c.authenticatedDesc,
+ c.dhtDesc, c.biscuitExpiryDesc, c.servicesDesc,
+ } {
+ ch <- d
+ }
+}
+
+func (c *nodeStateCollector) Collect(ch chan<- prometheus.Metric) {
+ n := c.n
+ if n.debugReady() != nil {
+ ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 0)
+ return
+ }
+ ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 1)
+
+ connected := 0.0
+ if n.IsConnected() {
+ connected = 1
+ }
+ ch <- prometheus.MustNewConstMetric(c.meshConnectedDesc, prometheus.GaugeValue, connected)
+ ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(n.Host.Network().Peers())))
+
+ authenticated := 0
+ n.authPeers.Range(func(_, _ any) bool {
+ authenticated++
+ return true
+ })
+ ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(authenticated))
+ ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(n.DHT.RoutingTable().Size()))
+
+ if exp, err := n.Store.LoadIdentityExpiration(); err == nil && exp > 0 {
+ ch <- prometheus.MustNewConstMetric(c.biscuitExpiryDesc, prometheus.GaugeValue, float64(exp))
+ }
+
+ if n.services != nil {
+ byType := map[string]int{}
+ for _, svc := range n.services.List(api.ServiceType_SERVICE_TYPE_UNSPECIFIED) {
+ byType[serviceTypeLabel(svc.Type)]++
+ }
+ for t, count := range byType {
+ ch <- prometheus.MustNewConstMetric(c.servicesDesc, prometheus.GaugeValue, float64(count), t)
+ }
+ }
+}
+
+// serviceTypeLabel turns SERVICE_TYPE_MCP into "mcp".
+func serviceTypeLabel(t api.ServiceType) string {
+ return strings.ToLower(strings.TrimPrefix(t.String(), "SERVICE_TYPE_"))
+}
+
+// metricsHandler serves the process-wide registry, which carries the request
+// and inference counters and the Go runtime, alongside this node's own state.
+// The state collector is per node so two nodes in one process (tests,
+// sam-one) never fight over a registration.
+func (n *SamNode) metricsHandler() http.Handler {
+ n.metricsOnce.Do(func() {
+ n.metricsRegistry = prometheus.NewRegistry()
+ n.metricsRegistry.MustRegister(newNodeStateCollector(n))
+ })
+ return promhttp.HandlerFor(
+ prometheus.Gatherers{prometheus.DefaultGatherer, n.metricsRegistry},
+ promhttp.HandlerOpts{},
+ )
+}
+
+// StartMetricsServer serves /metrics, /healthz and /readyz on addr with no
+// authentication, for a scraper and a kubelet that hold no sidecar token.
+// The sidecar's own /metrics stays token-gated: this listener exists so a
+// socket-only node, which has no TCP port at all, can still be observed. It
+// is off unless an operator names the address, since nothing on it is gated.
+func StartMetricsServer(node *SamNode, addr string) (*http.Server, error) {
+ if addr == "" {
+ return nil, errors.New("no metrics address configured")
+ }
+ listener, err := net.Listen("tcp", addr)
+ if err != nil {
+ return nil, err
+ }
+
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", node.metricsHandler())
+ mux.HandleFunc("/healthz", handleHealthz)
+ mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
+ if node.debugReady() != nil || !node.IsConnected() {
+ http.Error(w, "not connected to the mesh", http.StatusServiceUnavailable)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ })
+
+ server := &http.Server{
+ // Informational once Serve has the listener; lets callers find the port.
+ Addr: listener.Addr().String(),
+ Handler: mux,
+ ReadHeaderTimeout: 10 * time.Second,
+ }
+ go func() {
+ _ = server.Serve(listener)
+ }()
+ logger.Infof("Metrics listening on http://%s", listener.Addr())
+ return server, nil
+}
diff --git a/internal/node/node_metrics_test.go b/internal/node/node_metrics_test.go
new file mode 100644
index 00000000..5747f9ed
--- /dev/null
+++ b/internal/node/node_metrics_test.go
@@ -0,0 +1,150 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+func getMetricsBody(t *testing.T, url string) (int, string) {
+ t.Helper()
+ resp, err := http.Get(url)
+ if err != nil {
+ t.Fatalf("GET %s: %v", url, err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ return resp.StatusCode, string(body)
+}
+
+func TestNodeMetricsServer(t *testing.T) {
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+
+ // Half-built, as a node is between NewSamNode and Start: alive, not
+ // ready, and nothing read from a host that does not exist yet.
+ node := &SamNode{Store: store, services: NewServiceRegistry(&fakeDHT{}, 0)}
+ srv, err := StartMetricsServer(node, "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = srv.Close() }()
+ base := "http://" + srv.Addr
+ if base == "http://" {
+ t.Fatal("metrics server did not record its address")
+ }
+
+ if code, _ := getMetricsBody(t, base+"/healthz"); code != http.StatusOK {
+ t.Errorf("/healthz = %d before ready", code)
+ }
+ if code, _ := getMetricsBody(t, base+"/readyz"); code != http.StatusServiceUnavailable {
+ t.Errorf("/readyz = %d before ready, want 503", code)
+ }
+ code, body := getMetricsBody(t, base+"/metrics")
+ if code != http.StatusOK {
+ t.Fatalf("/metrics = %d: %s", code, body)
+ }
+ if !strings.Contains(body, "sam_node_ready 0") {
+ t.Error("/metrics missing sam_node_ready 0 before start")
+ }
+ if strings.Contains(body, "sam_node_connected_peers") {
+ t.Error("/metrics exported host state before the host existed")
+ }
+
+ host, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = host.Close() }()
+ kad, err := dht.New(host, dht.Mode(dht.ModeServer))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = kad.Close() }()
+
+ expiry := time.Now().Add(time.Hour).Unix()
+ if err := store.SaveIdentityExpiration(expiry); err != nil {
+ t.Fatal(err)
+ }
+ node.Host = host
+ node.DHT = kad
+ node.authPeers.Store(peer.ID("peer-a"), time.Now().Add(time.Hour))
+ node.authPeers.Store(peer.ID("peer-b"), time.Now().Add(time.Hour))
+
+ // Ready, but no authenticated router yet: the kubelet must not route to it.
+ if code, _ := getMetricsBody(t, base+"/readyz"); code != http.StatusServiceUnavailable {
+ t.Errorf("/readyz = %d with no router, want 503", code)
+ }
+ _, body = getMetricsBody(t, base+"/metrics")
+ for _, want := range []string{
+ "sam_node_ready 1",
+ "sam_node_mesh_connected 0",
+ "sam_node_connected_peers 0",
+ "sam_node_authenticated_peers 2",
+ "sam_node_dht_routing_table_size 0",
+ "sam_node_biscuit_expiry_timestamp_seconds " + strconv.FormatFloat(float64(expiry), 'g', -1, 64),
+ // The process-wide registry rides along: sidecar counters and libp2p.
+ "sam_node_requests_in_flight",
+ "libp2p_",
+ } {
+ if !strings.Contains(body, want) {
+ t.Errorf("/metrics missing %q", want)
+ }
+ }
+ if strings.Contains(body, "sam_node_services_registered") {
+ t.Error("services gauge exported with an empty registry")
+ }
+
+ // An authenticated router that is actually connected flips both.
+ routerHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = routerHost.Close() }()
+ if err := host.Connect(t.Context(), peer.AddrInfo{ID: routerHost.ID(), Addrs: routerHost.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+ node.mu.Lock()
+ node.authenticatedRouters = map[peer.ID]bool{routerHost.ID(): true}
+ node.mu.Unlock()
+
+ if code, _ := getMetricsBody(t, base+"/readyz"); code != http.StatusOK {
+ t.Errorf("/readyz = %d once a router is authenticated", code)
+ }
+ _, body = getMetricsBody(t, base+"/metrics")
+ for _, want := range []string{"sam_node_mesh_connected 1", "sam_node_connected_peers 1"} {
+ if !strings.Contains(body, want) {
+ t.Errorf("/metrics missing %q", want)
+ }
+ }
+}
+
+func TestStartMetricsServerRequiresAddress(t *testing.T) {
+ if _, err := StartMetricsServer(&SamNode{}, ""); err == nil {
+ t.Fatal("expected an error for an empty address")
+ }
+}
diff --git a/internal/node/sidecar.go b/internal/node/sidecar.go
index ff3c5cac..ea6ca1e1 100644
--- a/internal/node/sidecar.go
+++ b/internal/node/sidecar.go
@@ -36,7 +36,6 @@ import (
libp2phttp "github.com/libp2p/go-libp2p-http"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
- "github.com/prometheus/client_golang/prometheus/promhttp"
)
// StartSidecarServer serves the node's local API on a TCP address, on a Unix
@@ -51,7 +50,7 @@ func StartSidecarServer(node *SamNode, addr, socketPath, token, certFile, keyFil
// Gated like the rest: the labels carry peer IDs and per-peer request counts,
// and this mux is reachable by any local process over TCP. Socket callers are
// unaffected, which is how every scrape in this repo reads it.
- mux.Handle("/metrics", withAuth(token, true, promhttp.Handler()))
+ mux.Handle("/metrics", withAuth(token, true, node.metricsHandler()))
// Protected endpoints. allowAuthorizationFallback=true is safe here: none of
// these ever forward the inbound Authorization header to another service.