Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
add metrics
Antonio Ojea4eafec0a2407
diff --git a/cmd/sam-hub/hooks.go b/cmd/sam-hub/hooks.goindex 5e44d707..99ca2f60 100644--- a/cmd/sam-hub/hooks.go+++ b/cmd/sam-hub/hooks.go@@ -16,16 +16,16 @@ package mainimport ("context"- "encoding/json"- "fmt""sync""time"+ "github.com/google/sam/api""github.com/libp2p/go-libp2p/core/connmgr""github.com/libp2p/go-libp2p/core/control""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer""github.com/multiformats/go-multiaddr"+ "google.golang.org/protobuf/proto")var _ connmgr.ConnectionGater = (*hubConnGate)(nil)@@ -80,25 +80,33 @@ func (n *notifier) Disconnected(_ network.Network, c network.Conn) {delete(n.hub.gater.authenticated, p)delete(n.hub.gater.pending, p)n.hub.gater.mu.Unlock()- fmt.Printf("[Mesh] Peer %s disconnected. Authorization cleared.\n", p)++ logger.Infow("Peer disconnected", "peer_id", p, "was_authenticated", wasAuth)+if wasAuth {- event := MeshEvent{PeerID: p, Action: "LEFT"}- data, err := json.Marshal(event)- if err != nil {- fmt.Printf("Failed to marshal mesh event: %v\n", err)- return+ samHubActiveNodes.Dec() // Decrement active nodes++ event := &api.MeshEvent{+ Type: api.MeshEvent_EXIT,+ PeerId: p.String(),+ Timestamp: time.Now().Unix(),}- // Notify the mesh about the peer disconnection- if err := n.hub.EventTopic.Publish(context.Background(), data); err != nil {- fmt.Printf("Failed to publish mesh event: %v\n", err)- return+ // Sign event+ if err := n.hub.signEvent(event); err != nil {+ logger.Errorw("Failed to sign mesh event", "peer_id", p, "error", err)+ } else {+ data, err := proto.Marshal(event)+ if err != nil {+ logger.Errorw("Failed to marshal mesh event", "peer_id", p, "error", err)+ return+ }+ // Notify the mesh about the peer disconnection+ if err := n.hub.EventTopic.Publish(context.Background(), data); err != nil {+ logger.Errorw("Failed to publish mesh event", "peer_id", p, "error", err)+ return+ }+ logger.Infow("Published EXIT event", "peer_id", p)+ samHubMeshEventsTotal.WithLabelValues("EXIT").Inc()}- fmt.Printf("[Mesh] Peer %s left. Broadcasted departure to agents.\n", p)}}--// MeshEvent defines the message sent over Gossipsub-type MeshEvent struct {- PeerID peer.ID `json:"peer_id"`- Action string `json:"action"` // "JOINED", "LEFT", "BANNED"-}diff --git a/cmd/sam-hub/keyring.go b/cmd/sam-hub/keyring.goindex 70f1cc6f..a2677753 100644--- a/cmd/sam-hub/keyring.go+++ b/cmd/sam-hub/keyring.go@@ -87,10 +87,19 @@ func NewKeyRing(dbPath string, gracePeriod time.Duration, initialSeed []byte) (*}}} else if len(kr.Current.Private) == 0 {- if err := kr.rotate(gracePeriod); err != nil {+ pub, priv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {_ = db.Close()return nil, fmt.Errorf("failed to generate initial key: %w", err)}+ kr.Current = KeyPair{+ Private: priv,+ Public: pub,+ }+ if err := kr.save(); err != nil {+ _ = db.Close()+ return nil, fmt.Errorf("failed to save initial key: %w", err)+ }}return kr, nil@@ -118,26 +127,21 @@ func (kr *KeyRing) save() error {})}-// Rotate generates a new keypair and moves the current one to the previous list.-// It returns the new public key and the old private key for signing rotation events.-func (kr *KeyRing) Rotate(gracePeriod time.Duration) ([]byte, ed25519.PrivateKey, error) {- kr.mu.Lock()- defer kr.mu.Unlock()-- oldPriv := ed25519.PrivateKey(kr.Current.Private)+// PrepareRotation generates a new keypair and returns it along with the old private key for signing.+// It does NOT update the current key in the keyring yet.+func (kr *KeyRing) PrepareRotation() (newPub ed25519.PublicKey, newPriv ed25519.PrivateKey, oldPriv ed25519.PrivateKey, err error) {+ kr.mu.RLock()+ defer kr.mu.RUnlock()- err := kr.rotate(gracePeriod)- if err != nil {- return nil, nil, err- }- return kr.Current.Public, oldPriv, nil+ oldPriv = ed25519.PrivateKey(kr.Current.Private)+ newPub, newPriv, err = ed25519.GenerateKey(rand.Reader)+ return newPub, newPriv, oldPriv, err}-func (kr *KeyRing) rotate(gracePeriod time.Duration) error {- pub, priv, err := ed25519.GenerateKey(rand.Reader)- if err != nil {- return err- }+// CommitRotation promotes the new key to current and moves the old current to previous.+func (kr *KeyRing) CommitRotation(newPub ed25519.PublicKey, newPriv ed25519.PrivateKey, gracePeriod time.Duration) error {+ kr.mu.Lock()+ defer kr.mu.Unlock()if len(kr.Current.Private) > 0 {kr.Current.Expiration = time.Now().Add(gracePeriod)@@ -145,8 +149,8 @@ func (kr *KeyRing) rotate(gracePeriod time.Duration) error {}kr.Current = KeyPair{- Private: priv,- Public: pub,+ Private: newPriv,+ Public: newPub,}// Clean up expired keysdiff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.goindex 2ef015c8..7f7f912c 100644--- a/cmd/sam-hub/main.go+++ b/cmd/sam-hub/main.go@@ -27,7 +27,6 @@ import ("github.com/biscuit-auth/biscuit-go/v2""github.com/biscuit-auth/biscuit-go/v2/parser"- "golang.org/x/time/rate""github.com/coreos/go-oidc/v3/oidc""github.com/golang-jwt/jwt/v5""github.com/google/sam/api"@@ -37,11 +36,13 @@ import (pubsub "github.com/libp2p/go-libp2p-pubsub""github.com/libp2p/go-libp2p/core/host""github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/p2p/net/connmgr""github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic""github.com/libp2p/go-msgio"- "github.com/libp2p/go-libp2p/p2p/net/connmgr"+ "github.com/prometheus/client_golang/prometheus/promhttp"+ "golang.org/x/time/rate""github.com/libp2p/go-libp2p/p2p/transport/tcp""github.com/spf13/cobra"@@ -68,6 +69,7 @@ var (keyRotationInterval time.DurationkeyGracePeriod time.DurationkeysDBPath string+ metricsAddr string)var logger = golog.Logger("sam-hub")@@ -89,7 +91,7 @@ type Hub struct {// NewHub starts a host supporting both QUIC and TCP (with TLS 1.3)func NewHub(ctx context.Context, policy *api.PolicyConfig) (*Hub, error) {gater := newHubConnGate()-+// Connection Manager for DoS protectioncm, err := connmgr.NewConnManager(100, 400, connmgr.WithGracePeriod(time.Minute))if err != nil {@@ -191,31 +193,35 @@ func (h *Hub) handleEnroll(s network.Stream) {_ = s.Close()}()remotePeer := s.Conn().RemotePeer()-+if !h.limiter.Allow() {- logger.Warnf("[Enroll] Rate limit exceeded for %s, dropping connection", remotePeer)+ logger.Warnw("Rate limit exceeded during enrollment", "peer_id", remotePeer)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()_ = s.Reset()return}- logger.Infof("[Enroll] New enrollment request from %s", remotePeer)+ logger.Infow("New enrollment request", "peer_id", remotePeer)reader := msgio.NewVarintReaderSize(s, 1024*64)msg, err := reader.ReadMsg()if err != nil {- logger.Errorf("[Enroll] Failed to read message: %v", err)+ logger.Errorw("Failed to read enrollment message", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()return}defer reader.ReleaseMsg(msg)var req api.EnrollRequestif err := proto.Unmarshal(msg, &req); err != nil {- logger.Errorf("[Enroll] Failed to unmarshal request: %v", err)+ logger.Errorw("Failed to unmarshal enrollment request", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()return}if req.PeerId != remotePeer.String() {- logger.Warnf("[Enroll] Peer ID mismatch: %s != %s", req.PeerId, remotePeer)+ logger.Warnw("Peer ID mismatch in enrollment", "peer_id", remotePeer, "requested_peer_id", req.PeerId)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Peer ID mismatch", 0, nil, nil)return}@@ -224,13 +230,15 @@ func (h *Hub) handleEnroll(s network.Stream) {jwtParser := jwt.Parser{}jwtToken, _, err := jwtParser.ParseUnverified(req.Jwt, jwt.MapClaims{})if err != nil {- logger.Errorf("[Enroll] Failed to parse JWT: %v", err)+ logger.Errorw("Failed to parse JWT in enrollment", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Failed to parse JWT", 0, nil, nil)return}claims, ok := jwtToken.Claims.(jwt.MapClaims)if !ok {- logger.Warn("[Enroll] Invalid JWT claims")+ logger.Warnw("Invalid JWT claims in enrollment", "peer_id", remotePeer)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Invalid JWT claims", 0, nil, nil)return}@@ -247,21 +255,24 @@ func (h *Hub) handleEnroll(s network.Stream) {}if aud == "" {- logger.Warn("[Enroll] Missing aud claim")+ logger.Warnw("Missing aud claim in enrollment", "peer_id", remotePeer)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Missing aud claim", 0, nil, nil)return}alg, ok := jwtToken.Header["alg"].(string)if !ok || alg == "" {- logger.Warn("[Enroll] Missing alg header")+ logger.Warnw("Missing alg header in enrollment", "peer_id", remotePeer)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Missing alg header", 0, nil, nil)return}provider, ok := h.Providers[iss]if !ok {- logger.Warnf("[Enroll] Unknown issuer: %s", iss)+ logger.Warnw("Unknown issuer in enrollment", "peer_id", remotePeer, "issuer", iss)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, fmt.Sprintf("Unknown issuer: %s", iss), 0, nil, nil)return}@@ -270,18 +281,20 @@ func (h *Hub) handleEnroll(s network.Stream) {token, err := verifier.Verify(context.Background(), req.Jwt)if err != nil {- logger.Errorf("[Enroll] JWT validation failed: %v", err)+ logger.Errorw("JWT validation failed in enrollment", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, fmt.Sprintf("JWT validation failed: %v", err), 0, nil, nil)return}// Strictly enforce aud and alg after verificationif alg == "none" {- logger.Warnf("[Enroll] Invalid alg claim: %s", alg)+ logger.Warnw("Invalid alg claim in enrollment", "peer_id", remotePeer, "alg", alg)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Invalid alg claim", 0, nil, nil)return}- logger.Infof("[Enroll] JWT validated with aud: %s, alg: %s", aud, alg)+ logger.Infow("JWT validated", "peer_id", remotePeer, "aud", aud, "alg", alg)var roles []stringif rolesAny, ok := claims["roles"].([]any); ok {@@ -298,7 +311,8 @@ func (h *Hub) handleEnroll(s network.Stream) {Name: "expiration",IDs: []biscuit.Term{biscuit.Date(token.Expiry)},}}); err != nil {- logger.Errorf("[Enroll] Failed to add expiration fact: %v", err)+ logger.Errorw("Failed to add expiration fact to biscuit", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)return}@@ -307,7 +321,8 @@ func (h *Hub) handleEnroll(s network.Stream) {Name: "node",IDs: []biscuit.Term{biscuit.String(remotePeer.String())},}}); err != nil {- logger.Errorf("[Enroll] Failed to add node fact: %v", err)+ logger.Errorw("Failed to add node fact to biscuit", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)return}@@ -316,7 +331,8 @@ func (h *Hub) handleEnroll(s network.Stream) {Name: "client_peer_id",IDs: []biscuit.Term{biscuit.String(remotePeer.String())},}}); err != nil {- logger.Errorf("[Enroll] Failed to add client_peer_id fact: %v", err)+ logger.Errorw("Failed to add client_peer_id fact to biscuit", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)return}@@ -326,7 +342,8 @@ func (h *Hub) handleEnroll(s network.Stream) {Name: "group",IDs: []biscuit.Term{biscuit.String(role)},}}); err != nil {- logger.Errorf("[Enroll] Failed to add group fact: %v", err)+ logger.Errorw("Failed to add group fact to biscuit", "peer_id", remotePeer, "role", role, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)return}@@ -338,7 +355,7 @@ func (h *Hub) handleEnroll(s network.Stream) {Name: api.FactMCPTool,IDs: []biscuit.Term{biscuit.String(tool)},}}); err != nil {- logger.Errorf("[Enroll] Failed to add fact %s: %v", api.FactMCPTool, err)+ logger.Errorw("Failed to add MCP tool fact to biscuit", "peer_id", remotePeer, "tool", tool, "error", err)}}for _, target := range rolePolicy.Network.AllowedTargets {@@ -346,7 +363,7 @@ func (h *Hub) handleEnroll(s network.Stream) {Name: api.FactNetworkTarget,IDs: []biscuit.Term{biscuit.String(target)},}}); err != nil {- logger.Errorf("[Enroll] Failed to add fact %s: %v", api.FactNetworkTarget, err)+ logger.Errorw("Failed to add network target fact to biscuit", "peer_id", remotePeer, "target", target, "error", err)}}for _, customFact := range rolePolicy.CustomDatalog {@@ -357,16 +374,16 @@ func (h *Hub) handleEnroll(s network.Stream) {func() {defer func() {if r := recover(); r != nil {- logger.Errorf("[Enroll] Panic parsing custom fact %q: %v", trimmed, r)+ logger.Errorw("Panic parsing custom fact", "peer_id", remotePeer, "fact", trimmed, "recover", r)}}()fact, err := parser.FromStringFact(trimmed)if err != nil {- logger.Errorf("[Enroll] Failed to parse custom fact %q: %v", trimmed, err)+ logger.Errorw("Failed to parse custom fact", "peer_id", remotePeer, "fact", trimmed, "error", err)return}if err := builder.AddAuthorityFact(fact); err != nil {- logger.Errorf("[Enroll] Failed to add custom fact %s: %v", trimmed, err)+ logger.Errorw("Failed to add custom fact to biscuit", "peer_id", remotePeer, "fact", trimmed, "error", err)}}()}@@ -376,19 +393,25 @@ func (h *Hub) handleEnroll(s network.Stream) {t, err := builder.Build()if err != nil {- logger.Errorf("[Enroll] Failed to build biscuit: %v", err)+ logger.Errorw("Failed to build biscuit", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Failed to build biscuit", 0, nil, nil)return}biscuitData, err := t.Serialize()if err != nil {- logger.Errorf("[Enroll] Failed to serialize biscuit: %v", err)+ logger.Errorw("Failed to serialize biscuit", "peer_id", remotePeer, "error", err)+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()h.sendEnrollResponse(s, nil, "Failed to serialize biscuit", 0, nil, nil)return}h.gater.mu.Lock()+ // Update active nodes gauge if this is a new node+ if !h.gater.authenticated[remotePeer] {+ samHubActiveNodes.Inc()+ }h.gater.authenticated[remotePeer] = truedelete(h.gater.pending, remotePeer)@@ -404,7 +427,8 @@ func (h *Hub) handleEnroll(s network.Stream) {}h.sendEnrollResponse(s, biscuitData, "", token.Expiry.Unix(), knownPeers, policies)- logger.Infof("[Enroll] Successfully enrolled peer %s", remotePeer)+ logger.Infow("Successfully enrolled peer", "peer_id", remotePeer)+ samHubEnrollmentTotal.WithLabelValues("success").Inc()// Publish JOIN eventevent := &api.MeshEvent{@@ -413,16 +437,17 @@ func (h *Hub) handleEnroll(s network.Stream) {Timestamp: time.Now().Unix(),}if err := h.signEvent(event); err != nil {- logger.Errorf("[Enroll] Failed to sign event: %v", err)+ logger.Errorw("Failed to sign mesh event", "peer_id", remotePeer, "error", err)} else {eventData, err := proto.Marshal(event)if err != nil {- logger.Errorf("[Enroll] Failed to marshal event: %v", err)+ logger.Errorw("Failed to marshal mesh event", "peer_id", remotePeer, "error", err)} else {if err := h.EventTopic.Publish(context.Background(), eventData); err != nil {- logger.Errorf("[Enroll] Failed to publish event: %v", err)+ logger.Errorw("Failed to publish mesh event", "peer_id", remotePeer, "error", err)} else {- logger.Infof("[Enroll] Published JOIN event for %s", remotePeer)+ logger.Infow("Published JOIN event", "peer_id", remotePeer)+ samHubMeshEventsTotal.WithLabelValues("JOIN").Inc()}}}@@ -493,12 +518,13 @@ func (h *Hub) startRotation(ctx context.Context) {for {select {case <-ticker.C:- logger.Info("[Rotation] Rotating keys...")+ logger.Infow("Rotating keys")newPub, oldPriv, err := h.KeyRing.Rotate(keyGracePeriod)if err != nil {- logger.Errorf("[Rotation] Failed to rotate keys: %v", err)+ logger.Errorw("Failed to rotate keys", "error", err)continue}+ samHubKeyRotationsTotal.Inc()// Broadcast eventevent := &api.MeshEvent{@@ -506,25 +532,26 @@ func (h *Hub) startRotation(ctx context.Context) {Timestamp: time.Now().Unix(),NewPublicKey: newPub,}-+// Sign with the OLD key so nodes can verify it!event.Signature = nildata, err := proto.Marshal(event)if err != nil {- logger.Errorf("[Rotation] Failed to marshal event: %v", err)+ logger.Errorw("Failed to marshal key rotation event", "error", err)continue}event.Signature = ed25519.Sign(oldPriv, data)eventData, err := proto.Marshal(event)if err != nil {- logger.Errorf("[Rotation] Failed to marshal event: %v", err)+ logger.Errorw("Failed to marshal key rotation event", "error", err)continue}if err := h.EventTopic.Publish(ctx, eventData); err != nil {- logger.Errorf("[Rotation] Failed to publish event: %v", err)+ logger.Errorw("Failed to publish key rotation event", "error", err)} else {- logger.Infof("[Rotation] Broadcasted new public key: %x", newPub)+ logger.Infow("Broadcasted new public key", "public_key", hex.EncodeToString(newPub))+ samHubMeshEventsTotal.WithLabelValues("KEY_ROTATION").Inc()}case <-ctx.Done():return@@ -551,6 +578,9 @@ func main() {ctx := context.Background()// Initialize logging+ if os.Getenv("LOG_FORMAT") == "json" {+ _ = os.Setenv("GOLOG_LOG_FMT", "json")+ }golog.SetAllLoggers(golog.LevelInfo)if logLevel != "" {lvl, err := golog.LevelFromString(logLevel)@@ -576,6 +606,15 @@ func main() {// Watchdog: Expel peers that connect but never finish authenticationh.startWatchdog(ctx)+ go func() {+ mux := http.NewServeMux()+ mux.Handle("/metrics", promhttp.Handler())+ logger.Infof("Starting metrics server on %s", metricsAddr)+ if err := http.ListenAndServe(metricsAddr, mux); err != nil {+ logger.Errorf("Metrics server failed: %v", err)+ }+ }()+logger.Infof("SAM Hub Online (QUIC + TCP)")logger.Infof("MeshID: %s", h.MeshID)logger.Infof("PeerID: %s", h.Host.ID())@@ -600,6 +639,7 @@ func main() {rootCmd.Flags().DurationVar(&keyRotationInterval, "key-rotation-interval", 0, "Key rotation interval (e.g. 12h). 0 disables rotation.")rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 24*time.Hour, "Key grace period for old keys (e.g. 24h).")rootCmd.Flags().StringVar(&keysDBPath, "keys-db", "keys.db", "Path to BoltDB file for keys")+ rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", ":9090", "Address to listen on for Prometheus metrics")if err := rootCmd.Execute(); err != nil {os.Exit(1)diff --git a/cmd/sam-hub/metrics.go b/cmd/sam-hub/metrics.gonew file mode 100644index 00000000..77aecc5b--- /dev/null+++ b/cmd/sam-hub/metrics.go@@ -0,0 +1,28 @@+package main++import (+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+)++var (+ samHubActiveNodes = promauto.NewGauge(prometheus.GaugeOpts{+ Name: "sam_hub_active_nodes",+ Help: "The current number of nodes connected/enrolled in the mesh.",+ })++ samHubEnrollmentTotal = promauto.NewCounterVec(prometheus.CounterOpts{+ Name: "sam_hub_enrollment_total",+ Help: "Total enrollment requests, partitioned by status.",+ }, []string{"status"})++ samHubMeshEventsTotal = promauto.NewCounterVec(prometheus.CounterOpts{+ Name: "sam_hub_mesh_events_total",+ Help: "Total gossip events processed, partitioned by event_type.",+ }, []string{"event_type"})++ samHubKeyRotationsTotal = promauto.NewCounter(prometheus.CounterOpts{+ Name: "sam_hub_key_rotations_total",+ Help: "The number of times the Hub has rotated its cryptographic keys.",+ })+)