Skip to content

Agent Mesh

charts: metrics ports, HTTP probes and monitoring toggles

Antonio Ojeafbb8774b71d7
diff --git a/charts/sam-mesh/README.md b/charts/sam-mesh/README.md
index f8c67b84..807a2c79 100644
--- a/charts/sam-mesh/README.md
+++ b/charts/sam-mesh/README.md
@@ -117,3 +117,26 @@ gateway:
There is no bundled Dex. Point `controlPlane.oidcIssuer` at your identity
provider and register `https://<control-plane-hostname><consolePath>/auth/callback` as
a redirect URI for the OIDC client the control plane reports.
+
+## Metrics (`monitoring.*`)
+
+The control plane serves Prometheus metrics on its `http` port and the router
+on a dedicated `metrics` containerPort (`router.metricsPort`, default 9090),
+which also carries its `/healthz` and `/readyz` probes. Neither endpoint is
+authenticated, so the router port is deliberately never a hostPort or a
+Service.
+
+Scraping is off by default because both supported resources are CRDs:
+
+- `monitoring.podMonitoring.enabled: true` renders a
+ `monitoring.googleapis.com/v1` `PodMonitoring` per component for Google
+ Managed Prometheus, which GKE runs out of the box.
+- `monitoring.podMonitor.enabled: true` renders a `monitoring.coreos.com/v1`
+ `PodMonitor` per component for prometheus-operator; use
+ `monitoring.podMonitor.labels` for the selector your Prometheus matches on.
+
+Mesh size and state come from the control plane
+(`sam_control_plane_mesh_connected_peers`, `sam_control_plane_routers_active`,
+`sam_control_plane_enrolled_nodes{role,state}`); the router adds its live view
+(`sam_router_authenticated_peers`, `sam_router_auth_handshakes_total{result}`)
+alongside libp2p's own relay and connection metrics.
diff --git a/charts/sam-mesh/templates/monitoring.yaml b/charts/sam-mesh/templates/monitoring.yaml
new file mode 100644
index 00000000..455b0c54
--- /dev/null
+++ b/charts/sam-mesh/templates/monitoring.yaml
@@ -0,0 +1,48 @@
+{{- /*
+One scrape target per component, rendered for whichever collector the
+cluster runs. Pod-selecting resources are used for both so the router's
+metrics port never has to appear on a Service.
+*/ -}}
+{{- $targets := list (dict "name" "control-plane" "port" "http") -}}
+{{- if .Values.router.enabled -}}
+{{- $targets = append $targets (dict "name" "router" "port" "metrics") -}}
+{{- end -}}
+{{- range $t := $targets }}
+{{- if $.Values.monitoring.podMonitoring.enabled }}
+---
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+ name: {{ include "sam-mesh.fullname" $ }}-{{ $t.name }}
+ labels:
+ {{- include "sam-mesh.labels" $ | nindent 4 }}
+spec:
+ selector:
+ matchLabels:
+ app: {{ include "sam-mesh.fullname" $ }}-{{ $t.name }}
+ endpoints:
+ - port: {{ $t.port }}
+ path: /metrics
+ interval: {{ $.Values.monitoring.interval }}
+{{- end }}
+{{- if $.Values.monitoring.podMonitor.enabled }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+ name: {{ include "sam-mesh.fullname" $ }}-{{ $t.name }}
+ labels:
+ {{- include "sam-mesh.labels" $ | nindent 4 }}
+ {{- with $.Values.monitoring.podMonitor.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ selector:
+ matchLabels:
+ app: {{ include "sam-mesh.fullname" $ }}-{{ $t.name }}
+ podMetricsEndpoints:
+ - port: {{ $t.port }}
+ path: /metrics
+ interval: {{ $.Values.monitoring.interval }}
+{{- end }}
+{{- end }}
diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml
index ff7c9e88..532fda1d 100644
--- a/charts/sam-mesh/templates/router-statefulset.yaml
+++ b/charts/sam-mesh/templates/router-statefulset.yaml
@@ -87,13 +87,27 @@ spec:
hostPort: {{ .Values.router.hostPort }}
{{- end }}
protocol: UDP
+ # Cluster-internal: never given a hostPort, nothing on it is authenticated.
+ - containerPort: {{ .Values.router.metricsPort }}
+ name: metrics
+ protocol: TCP
+ # /healthz answers as soon as the process is up; /readyz once the
+ # router has enrolled and its libp2p host is online.
+ startupProbe:
+ httpGet:
+ path: /healthz
+ port: metrics
+ periodSeconds: 5
+ failureThreshold: 24
readinessProbe:
- tcpSocket:
- port: p2p-tcp
+ httpGet:
+ path: /readyz
+ port: metrics
periodSeconds: 5
livenessProbe:
- tcpSocket:
- port: p2p-tcp
+ httpGet:
+ path: /healthz
+ port: metrics
periodSeconds: 15
resources:
{{- toYaml .Values.router.resources | nindent 10 }}
@@ -112,6 +126,7 @@ spec:
- "--insecure-control-plane"
- "--listen=/ip4/0.0.0.0/tcp/4501"
- "--listen=/ip4/0.0.0.0/udp/4501/quic-v1"
+ - "--metrics-addr=0.0.0.0:{{ .Values.router.metricsPort }}"
{{- if .Values.router.externalAddrs }}
{{- range .Values.router.externalAddrs }}
- "--external-addr={{ . }}"
diff --git a/charts/sam-mesh/tests/monitoring_test.yaml b/charts/sam-mesh/tests/monitoring_test.yaml
new file mode 100644
index 00000000..90b5358e
--- /dev/null
+++ b/charts/sam-mesh/tests/monitoring_test.yaml
@@ -0,0 +1,84 @@
+suite: monitoring
+templates:
+ - templates/monitoring.yaml
+release:
+ # collapses fullname to "sam-mesh" (see _helpers.tpl)
+ name: sam-mesh
+tests:
+ - it: renders nothing by default, since both kinds are CRDs the cluster may lack
+ asserts:
+ - hasDocuments:
+ count: 0
+
+ - it: podMonitoring scrapes the control plane and router on their own ports
+ set:
+ monitoring.podMonitoring.enabled: true
+ asserts:
+ - hasDocuments:
+ count: 2
+ - isKind:
+ of: PodMonitoring
+ - isAPIVersion:
+ of: monitoring.googleapis.com/v1
+ - equal:
+ path: spec.selector.matchLabels.app
+ value: sam-mesh-control-plane
+ documentIndex: 0
+ - equal:
+ path: spec.endpoints[0].port
+ value: http
+ documentIndex: 0
+ - equal:
+ path: spec.selector.matchLabels.app
+ value: sam-mesh-router
+ documentIndex: 1
+ - equal:
+ path: spec.endpoints[0].port
+ value: metrics
+ documentIndex: 1
+ - equal:
+ path: spec.endpoints[0].interval
+ value: 30s
+ documentIndex: 1
+
+ - it: podMonitor renders the prometheus-operator shape with extra labels
+ set:
+ monitoring.podMonitor.enabled: true
+ monitoring.podMonitor.labels:
+ release: kube-prometheus-stack
+ monitoring.interval: 15s
+ asserts:
+ - hasDocuments:
+ count: 2
+ - isKind:
+ of: PodMonitor
+ - isAPIVersion:
+ of: monitoring.coreos.com/v1
+ - equal:
+ path: metadata.labels.release
+ value: kube-prometheus-stack
+ - equal:
+ path: spec.podMetricsEndpoints[0].path
+ value: /metrics
+ - equal:
+ path: spec.podMetricsEndpoints[0].interval
+ value: 15s
+
+ - it: skips the router target when the router is disabled
+ set:
+ monitoring.podMonitoring.enabled: true
+ router.enabled: false
+ asserts:
+ - hasDocuments:
+ count: 1
+ - equal:
+ path: spec.selector.matchLabels.app
+ value: sam-mesh-control-plane
+
+ - it: both collectors can be enabled at once
+ set:
+ monitoring.podMonitoring.enabled: true
+ monitoring.podMonitor.enabled: true
+ asserts:
+ - hasDocuments:
+ count: 4
diff --git a/charts/sam-mesh/tests/router-statefulset_test.yaml b/charts/sam-mesh/tests/router-statefulset_test.yaml
index 72636ddc..e38f9a74 100644
--- a/charts/sam-mesh/tests/router-statefulset_test.yaml
+++ b/charts/sam-mesh/tests/router-statefulset_test.yaml
@@ -154,3 +154,48 @@ tests:
- equal:
path: spec.template.spec.containers[0].image
value: sam-router:v9
+
+ - it: serves metrics on a containerPort that probes use and no hostPort reaches
+ set:
+ router.hostPort: 4501
+ documentSelector:
+ path: kind
+ value: StatefulSet
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metrics-addr=0.0.0.0:9090
+ - equal:
+ path: spec.template.spec.containers[0].ports[2].name
+ value: metrics
+ - equal:
+ path: spec.template.spec.containers[0].ports[2].containerPort
+ value: 9090
+ - notExists:
+ path: spec.template.spec.containers[0].ports[2].hostPort
+ - equal:
+ path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+ value: /readyz
+ - equal:
+ path: spec.template.spec.containers[0].readinessProbe.httpGet.port
+ value: metrics
+ - equal:
+ path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+ value: /healthz
+ - equal:
+ path: spec.template.spec.containers[0].startupProbe.httpGet.path
+ value: /healthz
+
+ - it: metricsPort moves the listener, the port and the probes together
+ set:
+ router.metricsPort: 9999
+ documentSelector:
+ path: kind
+ value: StatefulSet
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metrics-addr=0.0.0.0:9999
+ - equal:
+ path: spec.template.spec.containers[0].ports[2].containerPort
+ value: 9999
diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml
index c67b9d53..9a013dfa 100644
--- a/charts/sam-mesh/values.yaml
+++ b/charts/sam-mesh/values.yaml
@@ -143,6 +143,9 @@ router:
# Overrides the announced multiaddrs; derived from hostPort when left empty.
externalAddrs: []
allowLoopback: false
+ # Plain HTTP port for /metrics, /healthz and /readyz. Unauthenticated, so it
+ # is only ever a containerPort, never a hostPort or Service.
+ metricsPort: 9090
# PVC for /data/router.key, so the libp2p peer ID survives rescheduling.
storageSize: 1Gi
# PVC StorageClass; cluster default when null.
@@ -213,3 +216,18 @@ bootstrap:
# labels in its sam-node.yaml. Empty means nodes cannot enroll with any label
# (fail closed).
nodeLabels: []
+
+# Scraping of the control-plane and router /metrics endpoints. Both are
+# cluster-internal and unauthenticated; enable the resource for whichever
+# collector the cluster runs, since each is a CRD that may not be installed.
+monitoring:
+ interval: 30s
+ # Google Managed Prometheus (monitoring.googleapis.com/v1 PodMonitoring),
+ # present by default on GKE.
+ podMonitoring:
+ enabled: false
+ # prometheus-operator (monitoring.coreos.com/v1 PodMonitor).
+ podMonitor:
+ enabled: false
+ # Extra labels, e.g. the release selector a Prometheus instance matches on.
+ labels: {}
diff --git a/charts/sam-node/templates/deployment.yaml b/charts/sam-node/templates/deployment.yaml
index c314ae61..1868e9dc 100644
--- a/charts/sam-node/templates/deployment.yaml
+++ b/charts/sam-node/templates/deployment.yaml
@@ -46,9 +46,37 @@ spec:
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--api-token-path=/var/run/secrets/sam/api-token"
- "--bind-addr={{ .Values.bindAddr }}"
+ {{- if .Values.metricsPort }}
+ - "--metrics-addr=0.0.0.0:{{ .Values.metricsPort }}"
+ {{- end }}
{{- range .Values.extraArgs }}
- {{ . | quote }}
{{- end }}
+ {{- if .Values.metricsPort }}
+ ports:
+ # Cluster-internal and unauthenticated; the API token is never needed here.
+ - containerPort: {{ .Values.metricsPort }}
+ name: metrics
+ protocol: TCP
+ # /healthz answers as soon as the process is up; /readyz once the node
+ # holds an authenticated connection to a router.
+ startupProbe:
+ httpGet:
+ path: /healthz
+ port: metrics
+ periodSeconds: 5
+ failureThreshold: 24
+ readinessProbe:
+ httpGet:
+ path: /readyz
+ port: metrics
+ periodSeconds: 10
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: metrics
+ periodSeconds: 20
+ {{- end }}
{{- with .Values.resources }}
resources:
{{- toYaml . | nindent 12 }}
diff --git a/charts/sam-node/templates/monitoring.yaml b/charts/sam-node/templates/monitoring.yaml
new file mode 100644
index 00000000..02bbc13c
--- /dev/null
+++ b/charts/sam-node/templates/monitoring.yaml
@@ -0,0 +1,39 @@
+{{- if .Values.metricsPort }}
+{{- if .Values.monitoring.podMonitoring.enabled }}
+---
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+ name: {{ include "sam-node.fullname" . }}
+ labels:
+ {{- include "sam-node.labels" . | nindent 4 }}
+spec:
+ selector:
+ matchLabels:
+ {{- include "sam-node.selectorLabels" . | nindent 6 }}
+ endpoints:
+ - port: metrics
+ path: /metrics
+ interval: {{ .Values.monitoring.interval }}
+{{- end }}
+{{- if .Values.monitoring.podMonitor.enabled }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+ name: {{ include "sam-node.fullname" . }}
+ labels:
+ {{- include "sam-node.labels" . | nindent 4 }}
+ {{- with .Values.monitoring.podMonitor.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ selector:
+ matchLabels:
+ {{- include "sam-node.selectorLabels" . | nindent 6 }}
+ podMetricsEndpoints:
+ - port: metrics
+ path: /metrics
+ interval: {{ .Values.monitoring.interval }}
+{{- end }}
+{{- end }}
diff --git a/charts/sam-node/tests/deployment_test.yaml b/charts/sam-node/tests/deployment_test.yaml
index 62289eb7..cd8a4ceb 100644
--- a/charts/sam-node/tests/deployment_test.yaml
+++ b/charts/sam-node/tests/deployment_test.yaml
@@ -125,3 +125,48 @@ tests:
- equal:
path: spec.template.spec.serviceAccountName
value: existing-sa
+
+ - it: serves metrics on a containerPort the probes use, apart from the API
+ template: templates/deployment.yaml
+ set:
+ controlPlaneUrl: http://sam-mesh-control-plane:8080
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: "--metrics-addr=0.0.0.0:9090"
+ - equal:
+ path: spec.template.spec.containers[0].ports[0].name
+ value: metrics
+ - equal:
+ path: spec.template.spec.containers[0].ports[0].containerPort
+ value: 9090
+ - equal:
+ path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+ value: /readyz
+ - equal:
+ path: spec.template.spec.containers[0].readinessProbe.httpGet.port
+ value: metrics
+ - equal:
+ path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+ value: /healthz
+ - equal:
+ path: spec.template.spec.containers[0].startupProbe.httpGet.path
+ value: /healthz
+
+ - it: metricsPort 0 drops the listener, the port and the probes together
+ template: templates/deployment.yaml
+ set:
+ controlPlaneUrl: http://sam-mesh-control-plane:8080
+ metricsPort: 0
+ asserts:
+ - notContains:
+ path: spec.template.spec.containers[0].args
+ content: "--metrics-addr=0.0.0.0:9090"
+ - notExists:
+ path: spec.template.spec.containers[0].ports
+ - notExists:
+ path: spec.template.spec.containers[0].readinessProbe
+ - notExists:
+ path: spec.template.spec.containers[0].livenessProbe
+ - notExists:
+ path: spec.template.spec.containers[0].startupProbe
diff --git a/charts/sam-node/tests/monitoring_test.yaml b/charts/sam-node/tests/monitoring_test.yaml
new file mode 100644
index 00000000..09b116ae
--- /dev/null
+++ b/charts/sam-node/tests/monitoring_test.yaml
@@ -0,0 +1,61 @@
+suite: sam-node monitoring
+templates:
+ - templates/monitoring.yaml
+release:
+ name: sam-node
+tests:
+ - it: renders nothing by default, since both kinds are CRDs the cluster may lack
+ set:
+ controlPlaneUrl: http://sam-mesh-control-plane:8080
+ asserts:
+ - hasDocuments:
+ count: 0
+
+ - it: podMonitoring selects the node pods on the metrics port
+ set:
+ controlPlaneUrl: http://sam-mesh-control-plane:8080
+ monitoring.podMonitoring.enabled: true
+ asserts:
+ - hasDocuments:
+ count: 1
+ - isKind:
+ of: PodMonitoring
+ - isAPIVersion:
+ of: monitoring.googleapis.com/v1
+ - equal:
+ path: spec.selector.matchLabels["app.kubernetes.io/name"]
+ value: sam-node
+ - equal:
+ path: spec.endpoints[0].port
+ value: metrics
+ - equal:
+ path: spec.endpoints[0].interval
+ value: 30s
+
+ - it: podMonitor renders the prometheus-operator shape with extra labels
+ set:
+ controlPlaneUrl: http://sam-mesh-control-plane:8080
+ monitoring.podMonitor.enabled: true
+ monitoring.podMonitor.labels:
+ release: kube-prometheus-stack
+ asserts:
+ - hasDocuments:
+ count: 1
+ - isKind:
+ of: PodMonitor
+ - equal:
+ path: metadata.labels.release
+ value: kube-prometheus-stack
+ - equal:
+ path: spec.podMetricsEndpoints[0].port
+ value: metrics
+
+ - it: renders nothing when the metrics listener is disabled
+ set:
+ controlPlaneUrl: http://sam-mesh-control-plane:8080
+ metricsPort: 0
+ monitoring.podMonitoring.enabled: true
+ monitoring.podMonitor.enabled: true
+ asserts:
+ - hasDocuments:
+ count: 0
diff --git a/charts/sam-node/values.yaml b/charts/sam-node/values.yaml
index 9f8510c8..64d3d075 100644
--- a/charts/sam-node/values.yaml
+++ b/charts/sam-node/values.yaml
@@ -43,6 +43,25 @@ securityContext:
# pod-private; set 0.0.0.0:8080 to expose it on the pod IP.
bindAddr: "127.0.0.1:8080"
+# Plain HTTP port for /metrics, /healthz and /readyz, separate from bindAddr
+# so neither the scraper nor the kubelet needs the API token. Unauthenticated,
+# so it is only ever a containerPort. Set to 0 to disable it and the probes.
+metricsPort: 9090
+
+# Scraping of the metrics port. Each is a CRD the cluster may not have, so
+# both are off until you pick the collector you run.
+monitoring:
+ interval: 30s
+ # Google Managed Prometheus (monitoring.googleapis.com/v1 PodMonitoring),
+ # present by default on GKE.
+ podMonitoring:
+ enabled: false
+ # prometheus-operator (monitoring.coreos.com/v1 PodMonitor).
+ podMonitor:
+ enabled: false
+ # Extra labels, e.g. the release selector a Prometheus instance matches on.
+ labels: {}
+
# Extra sam-node args, e.g. ["--discovery-interval=200ms"].
extraArgs: []