Skip to content

Agent Mesh

add metrics

Antonio Ojea4eafec0a2407
diff --git a/cmd/sam-hub/hooks.go b/cmd/sam-hub/hooks.go
index 5e44d707..99ca2f60 100644
--- a/cmd/sam-hub/hooks.go
+++ b/cmd/sam-hub/hooks.go
@@ -16,16 +16,16 @@ package main
import (
"context"
- "encoding/json"
- "fmt"
"sync"
"time"
+ "github.com/google/sam/api"
"github.com/libp2p/go-libp2p/core/connmgr"
"github.com/libp2p/go-libp2p/core/control"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multiaddr"
+ "google.golang.org/protobuf/proto"
)
var _ connmgr.ConnectionGater = (*hubConnGate)(nil)
@@ -80,25 +80,33 @@ func (n *notifier) Disconnected(_ network.Network, c network.Conn) {
delete(n.hub.gater.authenticated, p)
delete(n.hub.gater.pending, p)
n.hub.gater.mu.Unlock()
- fmt.Printf("[Mesh] Peer %s disconnected. Authorization cleared.\n", p)
+
+ logger.Infow("Peer disconnected", "peer_id", p, "was_authenticated", wasAuth)
+
if wasAuth {
- event := MeshEvent{PeerID: p, Action: "LEFT"}
- data, err := json.Marshal(event)
- if err != nil {
- fmt.Printf("Failed to marshal mesh event: %v\n", err)
- return
+ samHubActiveNodes.Dec() // Decrement active nodes
+
+ event := &api.MeshEvent{
+ Type: api.MeshEvent_EXIT,
+ PeerId: p.String(),
+ Timestamp: time.Now().Unix(),
}
- // Notify the mesh about the peer disconnection
- if err := n.hub.EventTopic.Publish(context.Background(), data); err != nil {
- fmt.Printf("Failed to publish mesh event: %v\n", err)
- return
+ // Sign event
+ if err := n.hub.signEvent(event); err != nil {
+ logger.Errorw("Failed to sign mesh event", "peer_id", p, "error", err)
+ } else {
+ data, err := proto.Marshal(event)
+ if err != nil {
+ logger.Errorw("Failed to marshal mesh event", "peer_id", p, "error", err)
+ return
+ }
+ // Notify the mesh about the peer disconnection
+ if err := n.hub.EventTopic.Publish(context.Background(), data); err != nil {
+ logger.Errorw("Failed to publish mesh event", "peer_id", p, "error", err)
+ return
+ }
+ logger.Infow("Published EXIT event", "peer_id", p)
+ samHubMeshEventsTotal.WithLabelValues("EXIT").Inc()
}
- fmt.Printf("[Mesh] Peer %s left. Broadcasted departure to agents.\n", p)
}
}
-
-// MeshEvent defines the message sent over Gossipsub
-type MeshEvent struct {
- PeerID peer.ID `json:"peer_id"`
- Action string `json:"action"` // "JOINED", "LEFT", "BANNED"
-}
diff --git a/cmd/sam-hub/keyring.go b/cmd/sam-hub/keyring.go
index 70f1cc6f..a2677753 100644
--- a/cmd/sam-hub/keyring.go
+++ b/cmd/sam-hub/keyring.go
@@ -87,10 +87,19 @@ func NewKeyRing(dbPath string, gracePeriod time.Duration, initialSeed []byte) (*
}
}
} else if len(kr.Current.Private) == 0 {
- if err := kr.rotate(gracePeriod); err != nil {
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
_ = db.Close()
return nil, fmt.Errorf("failed to generate initial key: %w", err)
}
+ kr.Current = KeyPair{
+ Private: priv,
+ Public: pub,
+ }
+ if err := kr.save(); err != nil {
+ _ = db.Close()
+ return nil, fmt.Errorf("failed to save initial key: %w", err)
+ }
}
return kr, nil
@@ -118,26 +127,21 @@ func (kr *KeyRing) save() error {
})
}
-// Rotate generates a new keypair and moves the current one to the previous list.
-// It returns the new public key and the old private key for signing rotation events.
-func (kr *KeyRing) Rotate(gracePeriod time.Duration) ([]byte, ed25519.PrivateKey, error) {
- kr.mu.Lock()
- defer kr.mu.Unlock()
-
- oldPriv := ed25519.PrivateKey(kr.Current.Private)
+// PrepareRotation generates a new keypair and returns it along with the old private key for signing.
+// It does NOT update the current key in the keyring yet.
+func (kr *KeyRing) PrepareRotation() (newPub ed25519.PublicKey, newPriv ed25519.PrivateKey, oldPriv ed25519.PrivateKey, err error) {
+ kr.mu.RLock()
+ defer kr.mu.RUnlock()
- err := kr.rotate(gracePeriod)
- if err != nil {
- return nil, nil, err
- }
- return kr.Current.Public, oldPriv, nil
+ oldPriv = ed25519.PrivateKey(kr.Current.Private)
+ newPub, newPriv, err = ed25519.GenerateKey(rand.Reader)
+ return newPub, newPriv, oldPriv, err
}
-func (kr *KeyRing) rotate(gracePeriod time.Duration) error {
- pub, priv, err := ed25519.GenerateKey(rand.Reader)
- if err != nil {
- return err
- }
+// CommitRotation promotes the new key to current and moves the old current to previous.
+func (kr *KeyRing) CommitRotation(newPub ed25519.PublicKey, newPriv ed25519.PrivateKey, gracePeriod time.Duration) error {
+ kr.mu.Lock()
+ defer kr.mu.Unlock()
if len(kr.Current.Private) > 0 {
kr.Current.Expiration = time.Now().Add(gracePeriod)
@@ -145,8 +149,8 @@ func (kr *KeyRing) rotate(gracePeriod time.Duration) error {
}
kr.Current = KeyPair{
- Private: priv,
- Public: pub,
+ Private: newPriv,
+ Public: newPub,
}
// Clean up expired keys
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index 2ef015c8..7f7f912c 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -27,7 +27,6 @@ import (
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/biscuit-auth/biscuit-go/v2/parser"
- "golang.org/x/time/rate"
"github.com/coreos/go-oidc/v3/oidc"
"github.com/golang-jwt/jwt/v5"
"github.com/google/sam/api"
@@ -37,11 +36,13 @@ import (
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
"github.com/libp2p/go-msgio"
- "github.com/libp2p/go-libp2p/p2p/net/connmgr"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+ "golang.org/x/time/rate"
"github.com/libp2p/go-libp2p/p2p/transport/tcp"
"github.com/spf13/cobra"
@@ -68,6 +69,7 @@ var (
keyRotationInterval time.Duration
keyGracePeriod time.Duration
keysDBPath string
+ metricsAddr string
)
var logger = golog.Logger("sam-hub")
@@ -89,7 +91,7 @@ type Hub struct {
// NewHub starts a host supporting both QUIC and TCP (with TLS 1.3)
func NewHub(ctx context.Context, policy *api.PolicyConfig) (*Hub, error) {
gater := newHubConnGate()
-
+
// Connection Manager for DoS protection
cm, err := connmgr.NewConnManager(100, 400, connmgr.WithGracePeriod(time.Minute))
if err != nil {
@@ -191,31 +193,35 @@ func (h *Hub) handleEnroll(s network.Stream) {
_ = s.Close()
}()
remotePeer := s.Conn().RemotePeer()
-
+
if !h.limiter.Allow() {
- logger.Warnf("[Enroll] Rate limit exceeded for %s, dropping connection", remotePeer)
+ logger.Warnw("Rate limit exceeded during enrollment", "peer_id", remotePeer)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
_ = s.Reset()
return
}
- logger.Infof("[Enroll] New enrollment request from %s", remotePeer)
+ logger.Infow("New enrollment request", "peer_id", remotePeer)
reader := msgio.NewVarintReaderSize(s, 1024*64)
msg, err := reader.ReadMsg()
if err != nil {
- logger.Errorf("[Enroll] Failed to read message: %v", err)
+ logger.Errorw("Failed to read enrollment message", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
return
}
defer reader.ReleaseMsg(msg)
var req api.EnrollRequest
if err := proto.Unmarshal(msg, &req); err != nil {
- logger.Errorf("[Enroll] Failed to unmarshal request: %v", err)
+ logger.Errorw("Failed to unmarshal enrollment request", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
return
}
if req.PeerId != remotePeer.String() {
- logger.Warnf("[Enroll] Peer ID mismatch: %s != %s", req.PeerId, remotePeer)
+ logger.Warnw("Peer ID mismatch in enrollment", "peer_id", remotePeer, "requested_peer_id", req.PeerId)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Peer ID mismatch", 0, nil, nil)
return
}
@@ -224,13 +230,15 @@ func (h *Hub) handleEnroll(s network.Stream) {
jwtParser := jwt.Parser{}
jwtToken, _, err := jwtParser.ParseUnverified(req.Jwt, jwt.MapClaims{})
if err != nil {
- logger.Errorf("[Enroll] Failed to parse JWT: %v", err)
+ logger.Errorw("Failed to parse JWT in enrollment", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Failed to parse JWT", 0, nil, nil)
return
}
claims, ok := jwtToken.Claims.(jwt.MapClaims)
if !ok {
- logger.Warn("[Enroll] Invalid JWT claims")
+ logger.Warnw("Invalid JWT claims in enrollment", "peer_id", remotePeer)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Invalid JWT claims", 0, nil, nil)
return
}
@@ -247,21 +255,24 @@ func (h *Hub) handleEnroll(s network.Stream) {
}
if aud == "" {
- logger.Warn("[Enroll] Missing aud claim")
+ logger.Warnw("Missing aud claim in enrollment", "peer_id", remotePeer)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Missing aud claim", 0, nil, nil)
return
}
alg, ok := jwtToken.Header["alg"].(string)
if !ok || alg == "" {
- logger.Warn("[Enroll] Missing alg header")
+ logger.Warnw("Missing alg header in enrollment", "peer_id", remotePeer)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Missing alg header", 0, nil, nil)
return
}
provider, ok := h.Providers[iss]
if !ok {
- logger.Warnf("[Enroll] Unknown issuer: %s", iss)
+ logger.Warnw("Unknown issuer in enrollment", "peer_id", remotePeer, "issuer", iss)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, fmt.Sprintf("Unknown issuer: %s", iss), 0, nil, nil)
return
}
@@ -270,18 +281,20 @@ func (h *Hub) handleEnroll(s network.Stream) {
token, err := verifier.Verify(context.Background(), req.Jwt)
if err != nil {
- logger.Errorf("[Enroll] JWT validation failed: %v", err)
+ logger.Errorw("JWT validation failed in enrollment", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, fmt.Sprintf("JWT validation failed: %v", err), 0, nil, nil)
return
}
// Strictly enforce aud and alg after verification
if alg == "none" {
- logger.Warnf("[Enroll] Invalid alg claim: %s", alg)
+ logger.Warnw("Invalid alg claim in enrollment", "peer_id", remotePeer, "alg", alg)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Invalid alg claim", 0, nil, nil)
return
}
- logger.Infof("[Enroll] JWT validated with aud: %s, alg: %s", aud, alg)
+ logger.Infow("JWT validated", "peer_id", remotePeer, "aud", aud, "alg", alg)
var roles []string
if rolesAny, ok := claims["roles"].([]any); ok {
@@ -298,7 +311,8 @@ func (h *Hub) handleEnroll(s network.Stream) {
Name: "expiration",
IDs: []biscuit.Term{biscuit.Date(token.Expiry)},
}}); err != nil {
- logger.Errorf("[Enroll] Failed to add expiration fact: %v", err)
+ logger.Errorw("Failed to add expiration fact to biscuit", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)
return
}
@@ -307,7 +321,8 @@ func (h *Hub) handleEnroll(s network.Stream) {
Name: "node",
IDs: []biscuit.Term{biscuit.String(remotePeer.String())},
}}); err != nil {
- logger.Errorf("[Enroll] Failed to add node fact: %v", err)
+ logger.Errorw("Failed to add node fact to biscuit", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)
return
}
@@ -316,7 +331,8 @@ func (h *Hub) handleEnroll(s network.Stream) {
Name: "client_peer_id",
IDs: []biscuit.Term{biscuit.String(remotePeer.String())},
}}); err != nil {
- logger.Errorf("[Enroll] Failed to add client_peer_id fact: %v", err)
+ logger.Errorw("Failed to add client_peer_id fact to biscuit", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)
return
}
@@ -326,7 +342,8 @@ func (h *Hub) handleEnroll(s network.Stream) {
Name: "group",
IDs: []biscuit.Term{biscuit.String(role)},
}}); err != nil {
- logger.Errorf("[Enroll] Failed to add group fact: %v", err)
+ logger.Errorw("Failed to add group fact to biscuit", "peer_id", remotePeer, "role", role, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Failed to mint biscuit", 0, nil, nil)
return
}
@@ -338,7 +355,7 @@ func (h *Hub) handleEnroll(s network.Stream) {
Name: api.FactMCPTool,
IDs: []biscuit.Term{biscuit.String(tool)},
}}); err != nil {
- logger.Errorf("[Enroll] Failed to add fact %s: %v", api.FactMCPTool, err)
+ logger.Errorw("Failed to add MCP tool fact to biscuit", "peer_id", remotePeer, "tool", tool, "error", err)
}
}
for _, target := range rolePolicy.Network.AllowedTargets {
@@ -346,7 +363,7 @@ func (h *Hub) handleEnroll(s network.Stream) {
Name: api.FactNetworkTarget,
IDs: []biscuit.Term{biscuit.String(target)},
}}); err != nil {
- logger.Errorf("[Enroll] Failed to add fact %s: %v", api.FactNetworkTarget, err)
+ logger.Errorw("Failed to add network target fact to biscuit", "peer_id", remotePeer, "target", target, "error", err)
}
}
for _, customFact := range rolePolicy.CustomDatalog {
@@ -357,16 +374,16 @@ func (h *Hub) handleEnroll(s network.Stream) {
func() {
defer func() {
if r := recover(); r != nil {
- logger.Errorf("[Enroll] Panic parsing custom fact %q: %v", trimmed, r)
+ logger.Errorw("Panic parsing custom fact", "peer_id", remotePeer, "fact", trimmed, "recover", r)
}
}()
fact, err := parser.FromStringFact(trimmed)
if err != nil {
- logger.Errorf("[Enroll] Failed to parse custom fact %q: %v", trimmed, err)
+ logger.Errorw("Failed to parse custom fact", "peer_id", remotePeer, "fact", trimmed, "error", err)
return
}
if err := builder.AddAuthorityFact(fact); err != nil {
- logger.Errorf("[Enroll] Failed to add custom fact %s: %v", trimmed, err)
+ logger.Errorw("Failed to add custom fact to biscuit", "peer_id", remotePeer, "fact", trimmed, "error", err)
}
}()
}
@@ -376,19 +393,25 @@ func (h *Hub) handleEnroll(s network.Stream) {
t, err := builder.Build()
if err != nil {
- logger.Errorf("[Enroll] Failed to build biscuit: %v", err)
+ logger.Errorw("Failed to build biscuit", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Failed to build biscuit", 0, nil, nil)
return
}
biscuitData, err := t.Serialize()
if err != nil {
- logger.Errorf("[Enroll] Failed to serialize biscuit: %v", err)
+ logger.Errorw("Failed to serialize biscuit", "peer_id", remotePeer, "error", err)
+ samHubEnrollmentTotal.WithLabelValues("failed").Inc()
h.sendEnrollResponse(s, nil, "Failed to serialize biscuit", 0, nil, nil)
return
}
h.gater.mu.Lock()
+ // Update active nodes gauge if this is a new node
+ if !h.gater.authenticated[remotePeer] {
+ samHubActiveNodes.Inc()
+ }
h.gater.authenticated[remotePeer] = true
delete(h.gater.pending, remotePeer)
@@ -404,7 +427,8 @@ func (h *Hub) handleEnroll(s network.Stream) {
}
h.sendEnrollResponse(s, biscuitData, "", token.Expiry.Unix(), knownPeers, policies)
- logger.Infof("[Enroll] Successfully enrolled peer %s", remotePeer)
+ logger.Infow("Successfully enrolled peer", "peer_id", remotePeer)
+ samHubEnrollmentTotal.WithLabelValues("success").Inc()
// Publish JOIN event
event := &api.MeshEvent{
@@ -413,16 +437,17 @@ func (h *Hub) handleEnroll(s network.Stream) {
Timestamp: time.Now().Unix(),
}
if err := h.signEvent(event); err != nil {
- logger.Errorf("[Enroll] Failed to sign event: %v", err)
+ logger.Errorw("Failed to sign mesh event", "peer_id", remotePeer, "error", err)
} else {
eventData, err := proto.Marshal(event)
if err != nil {
- logger.Errorf("[Enroll] Failed to marshal event: %v", err)
+ logger.Errorw("Failed to marshal mesh event", "peer_id", remotePeer, "error", err)
} else {
if err := h.EventTopic.Publish(context.Background(), eventData); err != nil {
- logger.Errorf("[Enroll] Failed to publish event: %v", err)
+ logger.Errorw("Failed to publish mesh event", "peer_id", remotePeer, "error", err)
} else {
- logger.Infof("[Enroll] Published JOIN event for %s", remotePeer)
+ logger.Infow("Published JOIN event", "peer_id", remotePeer)
+ samHubMeshEventsTotal.WithLabelValues("JOIN").Inc()
}
}
}
@@ -493,12 +518,13 @@ func (h *Hub) startRotation(ctx context.Context) {
for {
select {
case <-ticker.C:
- logger.Info("[Rotation] Rotating keys...")
+ logger.Infow("Rotating keys")
newPub, oldPriv, err := h.KeyRing.Rotate(keyGracePeriod)
if err != nil {
- logger.Errorf("[Rotation] Failed to rotate keys: %v", err)
+ logger.Errorw("Failed to rotate keys", "error", err)
continue
}
+ samHubKeyRotationsTotal.Inc()
// Broadcast event
event := &api.MeshEvent{
@@ -506,25 +532,26 @@ func (h *Hub) startRotation(ctx context.Context) {
Timestamp: time.Now().Unix(),
NewPublicKey: newPub,
}
-
+
// Sign with the OLD key so nodes can verify it!
event.Signature = nil
data, err := proto.Marshal(event)
if err != nil {
- logger.Errorf("[Rotation] Failed to marshal event: %v", err)
+ logger.Errorw("Failed to marshal key rotation event", "error", err)
continue
}
event.Signature = ed25519.Sign(oldPriv, data)
eventData, err := proto.Marshal(event)
if err != nil {
- logger.Errorf("[Rotation] Failed to marshal event: %v", err)
+ logger.Errorw("Failed to marshal key rotation event", "error", err)
continue
}
if err := h.EventTopic.Publish(ctx, eventData); err != nil {
- logger.Errorf("[Rotation] Failed to publish event: %v", err)
+ logger.Errorw("Failed to publish key rotation event", "error", err)
} else {
- logger.Infof("[Rotation] Broadcasted new public key: %x", newPub)
+ logger.Infow("Broadcasted new public key", "public_key", hex.EncodeToString(newPub))
+ samHubMeshEventsTotal.WithLabelValues("KEY_ROTATION").Inc()
}
case <-ctx.Done():
return
@@ -551,6 +578,9 @@ func main() {
ctx := context.Background()
// Initialize logging
+ if os.Getenv("LOG_FORMAT") == "json" {
+ _ = os.Setenv("GOLOG_LOG_FMT", "json")
+ }
golog.SetAllLoggers(golog.LevelInfo)
if logLevel != "" {
lvl, err := golog.LevelFromString(logLevel)
@@ -576,6 +606,15 @@ func main() {
// Watchdog: Expel peers that connect but never finish authentication
h.startWatchdog(ctx)
+ go func() {
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", promhttp.Handler())
+ logger.Infof("Starting metrics server on %s", metricsAddr)
+ if err := http.ListenAndServe(metricsAddr, mux); err != nil {
+ logger.Errorf("Metrics server failed: %v", err)
+ }
+ }()
+
logger.Infof("SAM Hub Online (QUIC + TCP)")
logger.Infof("MeshID: %s", h.MeshID)
logger.Infof("PeerID: %s", h.Host.ID())
@@ -600,6 +639,7 @@ func main() {
rootCmd.Flags().DurationVar(&keyRotationInterval, "key-rotation-interval", 0, "Key rotation interval (e.g. 12h). 0 disables rotation.")
rootCmd.Flags().DurationVar(&keyGracePeriod, "key-grace-period", 24*time.Hour, "Key grace period for old keys (e.g. 24h).")
rootCmd.Flags().StringVar(&keysDBPath, "keys-db", "keys.db", "Path to BoltDB file for keys")
+ rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", ":9090", "Address to listen on for Prometheus metrics")
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
diff --git a/cmd/sam-hub/metrics.go b/cmd/sam-hub/metrics.go
new file mode 100644
index 00000000..77aecc5b
--- /dev/null
+++ b/cmd/sam-hub/metrics.go
@@ -0,0 +1,28 @@
+package main
+
+import (
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+ samHubActiveNodes = promauto.NewGauge(prometheus.GaugeOpts{
+ Name: "sam_hub_active_nodes",
+ Help: "The current number of nodes connected/enrolled in the mesh.",
+ })
+
+ samHubEnrollmentTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+ Name: "sam_hub_enrollment_total",
+ Help: "Total enrollment requests, partitioned by status.",
+ }, []string{"status"})
+
+ samHubMeshEventsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+ Name: "sam_hub_mesh_events_total",
+ Help: "Total gossip events processed, partitioned by event_type.",
+ }, []string{"event_type"})
+
+ samHubKeyRotationsTotal = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sam_hub_key_rotations_total",
+ Help: "The number of times the Hub has rotated its cryptographic keys.",
+ })
+)