Skip to content

Agent Mesh

feat(obs): add /metrics, /healthz, /readyz endpoints and update k8s probes

Antonio Ojea8b52ae2e2e34
diff --git a/.github/k8s/sam-control-plane-template.yaml b/.github/k8s/sam-control-plane-template.yaml
index f7c3b621..da32e999 100644
--- a/.github/k8s/sam-control-plane-template.yaml
+++ b/.github/k8s/sam-control-plane-template.yaml
@@ -105,14 +105,14 @@ spec:
name: http
readinessProbe:
httpGet:
- path: /info
+ path: /readyz
port: http
scheme: HTTP
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
- path: /info
+ path: /healthz
port: http
scheme: HTTP
initialDelaySeconds: 15
diff --git a/charts/sam-mesh/templates/control-plane-deployment.yaml b/charts/sam-mesh/templates/control-plane-deployment.yaml
index ed381c81..43fb5a44 100644
--- a/charts/sam-mesh/templates/control-plane-deployment.yaml
+++ b/charts/sam-mesh/templates/control-plane-deployment.yaml
@@ -38,12 +38,12 @@ spec:
protocol: TCP
readinessProbe:
httpGet:
- path: /info
+ path: /readyz
port: http
periodSeconds: 5
livenessProbe:
httpGet:
- path: /info
+ path: /healthz
port: http
periodSeconds: 15
resources:
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 13a0d6dd..306d6b31 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -43,6 +43,7 @@ import (
golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/time/rate"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
@@ -147,6 +148,9 @@ func (s *Server) Start() error {
s.listener = l
mux := http.NewServeMux()
+ mux.HandleFunc("/healthz", s.HandleHealthz)
+ mux.HandleFunc("/readyz", s.HandleReadyz)
+ mux.Handle("/metrics", promhttp.Handler())
mux.HandleFunc("/info", s.HandleInfo)
mux.HandleFunc("/register", s.HandleRegister)
mux.HandleFunc("/keys", s.HandleKeys)
@@ -288,6 +292,36 @@ func (s *Server) runKeyRotationLoop() {
}
}
+// HandleHealthz HTTP GET `/healthz`
+func (s *Server) HandleHealthz(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"status":"ok"}`))
+}
+
+// HandleReadyz HTTP GET `/readyz`
+func (s *Server) HandleReadyz(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ if s.store != nil {
+ if err := s.store.Ping(r.Context()); err != nil {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusServiceUnavailable)
+ _, _ = w.Write([]byte(fmt.Sprintf(`{"status":"error","message":%q}`, err.Error())))
+ return
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"status":"ready"}`))
+}
+
// HandleInfo HTTP GET `/info`
func (s *Server) HandleInfo(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.go
index fe1c3b2f..1848e59f 100644
--- a/internal/controlplane/server_test.go
+++ b/internal/controlplane/server_test.go
@@ -156,6 +156,27 @@ func TestControlPlaneBasic(t *testing.T) {
client := &http.Client{Timeout: 5 * time.Second}
+ // Test /healthz
+ respHealth, err := client.Get(baseURL + "/healthz")
+ if err != nil || respHealth.StatusCode != http.StatusOK {
+ t.Fatalf("GET /healthz failed: %v, status: %v", err, respHealth.Status)
+ }
+ _ = respHealth.Body.Close()
+
+ // Test /readyz
+ respReady, err := client.Get(baseURL + "/readyz")
+ if err != nil || respReady.StatusCode != http.StatusOK {
+ t.Fatalf("GET /readyz failed: %v, status: %v", err, respReady.Status)
+ }
+ _ = respReady.Body.Close()
+
+ // Test /metrics
+ respMetrics, err := client.Get(baseURL + "/metrics")
+ if err != nil || respMetrics.StatusCode != http.StatusOK {
+ t.Fatalf("GET /metrics failed: %v, status: %v", err, respMetrics.Status)
+ }
+ _ = respMetrics.Body.Close()
+
// 1. Test /info (no routers registered yet)
resp, err := client.Get(baseURL + "/info")
if err != nil {
diff --git a/internal/node/sidecar.go b/internal/node/sidecar.go
index 738b16c2..3d4951e6 100644
--- a/internal/node/sidecar.go
+++ b/internal/node/sidecar.go
@@ -34,6 +34,7 @@ import (
libp2phttp "github.com/libp2p/go-libp2p-http"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
"google.golang.org/protobuf/encoding/protojson"
)
@@ -43,6 +44,7 @@ func StartSidecarServer(node *SamNode, addr, token, certFile, keyFile, caFile st
// Public endpoints
mux.HandleFunc("/healthz", handleHealthz)
mux.HandleFunc("/readyz", handleReadyz)
+ mux.Handle("/metrics", promhttp.Handler())
// Protected endpoints. allowAuthorizationFallback=true is safe here: none of
// these ever forward the inbound Authorization header to another service.
diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go
index ed3ad2be..bba9da7a 100644
--- a/internal/storage/sql_store.go
+++ b/internal/storage/sql_store.go
@@ -1181,6 +1181,14 @@ func (s *SQLStore) ListUsers(ctx context.Context) ([]User, error) {
return users, nil
}
+// Ping implements Store.
+func (s *SQLStore) Ping(ctx context.Context) error {
+ if s.db == nil {
+ return fmt.Errorf("database connection is nil")
+ }
+ return s.db.PingContext(ctx)
+}
+
// Close implements Store.
func (s *SQLStore) Close() error {
return s.db.Close()
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 0343c78f..cb122ded 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -94,6 +94,9 @@ type EnrollmentRequest struct {
// Store defines the persistent operations for the SAM control plane.
type Store interface {
+ // Ping checks the health of the underlying database connection.
+ Ping(ctx context.Context) error
+
// GetCurrentKey retrieves the active key pair for biscuit signing.
GetCurrentKey(ctx context.Context) (ed25519.PrivateKey, ed25519.PublicKey, error)