Skip to content

Agent Mesh

deploy: scrape the control plane, routers and canaries with Managed Prometheus

Antonio Ojeae9f423456c97
diff --git a/.github/k8s/sam-box-canary-template.yaml b/.github/k8s/sam-box-canary-template.yaml
index 0247d16b..476ab62a 100644
--- a/.github/k8s/sam-box-canary-template.yaml
+++ b/.github/k8s/sam-box-canary-template.yaml
@@ -59,6 +59,7 @@ spec:
metadata:
labels:
app: box-canary-${ENV_NAME}
+ sam-canary: "true"
spec:
serviceAccountName: sam-box-sa
# nano-init ships as its own image with no shell in it, so the binary is
@@ -84,6 +85,11 @@ spec:
# the socket's permissions are the credential.
- "--bind-addr="
- "--socket-path=/var/run/sam/node.sock"
+ # The one TCP port, and it carries nothing an API token would gate.
+ - "--metrics-addr=0.0.0.0:9090"
+ ports:
+ - containerPort: 9090
+ name: metrics
resources:
requests:
cpu: 50m
@@ -107,6 +113,10 @@ spec:
- "--sidecar-socket=/var/run/sam/node.sock"
- "--egress-allow=example.com"
- "--log-level=debug"
+ - "--metrics-addr=0.0.0.0:9091"
+ ports:
+ - containerPort: 9091
+ name: box-metrics
resources:
requests:
cpu: 20m
diff --git a/.github/k8s/sam-monitoring-template.yaml b/.github/k8s/sam-monitoring-template.yaml
new file mode 100644
index 00000000..d6dab1cc
--- /dev/null
+++ b/.github/k8s/sam-monitoring-template.yaml
@@ -0,0 +1,53 @@
+# Scrape targets for Google Managed Prometheus, which GKE runs by default.
+# A PodMonitoring only sees pods in its own namespace, so the canaries get
+# their own below.
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+ name: sam-control-plane-${ENV_NAME}
+ namespace: ${NAMESPACE}
+spec:
+ selector:
+ matchLabels:
+ app: sam-control-plane-${ENV_NAME}
+ endpoints:
+ - port: http
+ path: /metrics
+ interval: 30s
+---
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+ name: sam-router-${ENV_NAME}
+ namespace: ${NAMESPACE}
+spec:
+ selector:
+ matchLabels:
+ app: sam-router-${ENV_NAME}
+ endpoints:
+ - port: metrics
+ path: /metrics
+ interval: 30s
+---
+# Every canary pod carries sam-canary=true; a pod without one of these named
+# ports is simply not a target for that endpoint.
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+ name: sam-canaries-${ENV_NAME}
+ namespace: sam-canary-${ENV_NAME}
+spec:
+ selector:
+ matchLabels:
+ sam-canary: "true"
+ endpoints:
+ - port: metrics
+ path: /metrics
+ interval: 30s
+ - port: box-metrics
+ path: /metrics
+ interval: 30s
+ targetLabels:
+ fromPod:
+ - from: app
+ to: canary
diff --git a/.github/k8s/sam-node-cop-template.yaml b/.github/k8s/sam-node-cop-template.yaml
index 6a2d7474..29357c8b 100644
--- a/.github/k8s/sam-node-cop-template.yaml
+++ b/.github/k8s/sam-node-cop-template.yaml
@@ -30,6 +30,7 @@ spec:
metadata:
labels:
app: cop-canary-${ENV_NAME}
+ sam-canary: "true"
spec:
serviceAccountName: sam-node-sa
containers:
@@ -70,8 +71,11 @@ spec:
- "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
+ - "--metrics-addr=0.0.0.0:9090"
ports:
- containerPort: 8080
+ - containerPort: 9090
+ name: metrics
resources:
requests:
cpu: 50m
diff --git a/.github/k8s/sam-node-everything-template.yaml b/.github/k8s/sam-node-everything-template.yaml
index b79ac198..b5fa9ae1 100644
--- a/.github/k8s/sam-node-everything-template.yaml
+++ b/.github/k8s/sam-node-everything-template.yaml
@@ -30,6 +30,7 @@ spec:
metadata:
labels:
app: everything-canary-${ENV_NAME}
+ sam-canary: "true"
spec:
serviceAccountName: sam-node-sa
containers:
@@ -58,8 +59,11 @@ spec:
- "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
+ - "--metrics-addr=0.0.0.0:9090"
ports:
- containerPort: 8080
+ - containerPort: 9090
+ name: metrics
resources:
requests:
cpu: 50m
diff --git a/.github/k8s/sam-node-openclaw-template.yaml b/.github/k8s/sam-node-openclaw-template.yaml
index 64c65976..2ce0a7bf 100644
--- a/.github/k8s/sam-node-openclaw-template.yaml
+++ b/.github/k8s/sam-node-openclaw-template.yaml
@@ -47,6 +47,7 @@ spec:
metadata:
labels:
app: openclaw-canary-${ENV_NAME}
+ sam-canary: "true"
spec:
serviceAccountName: sam-node-sa
containers:
@@ -114,8 +115,11 @@ spec:
- "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
+ - "--metrics-addr=0.0.0.0:9090"
ports:
- containerPort: 8080
+ - containerPort: 9090
+ name: metrics
resources:
requests:
cpu: 50m
diff --git a/.github/k8s/sam-node-openrouter-template.yaml b/.github/k8s/sam-node-openrouter-template.yaml
index 578166a0..ae22d884 100644
--- a/.github/k8s/sam-node-openrouter-template.yaml
+++ b/.github/k8s/sam-node-openrouter-template.yaml
@@ -32,6 +32,7 @@ spec:
metadata:
labels:
app: openrouter-canary-${ENV_NAME}
+ sam-canary: "true"
spec:
serviceAccountName: sam-node-sa
containers:
@@ -67,8 +68,11 @@ spec:
- "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
+ - "--metrics-addr=0.0.0.0:9090"
ports:
- containerPort: 8080
+ - containerPort: 9090
+ name: metrics
resources:
requests:
cpu: 500m
diff --git a/.github/k8s/sam-node-template.yaml b/.github/k8s/sam-node-template.yaml
index 7b3bfdad..192fdcd2 100644
--- a/.github/k8s/sam-node-template.yaml
+++ b/.github/k8s/sam-node-template.yaml
@@ -37,6 +37,7 @@ spec:
metadata:
labels:
app: sam-canary-${ENV_NAME}
+ sam-canary: "true"
spec:
serviceAccountName: sam-node-sa
containers:
@@ -51,8 +52,12 @@ spec:
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
- "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
+ # 9091, not 9090: the agnhost container below owns 9090 in this pod.
+ - "--metrics-addr=0.0.0.0:9091"
ports:
- containerPort: 8080
+ - containerPort: 9091
+ name: metrics
resources:
requests:
cpu: 50m
diff --git a/.github/k8s/sam-node-vllm-template.yaml b/.github/k8s/sam-node-vllm-template.yaml
index a956e104..0a9dc86e 100644
--- a/.github/k8s/sam-node-vllm-template.yaml
+++ b/.github/k8s/sam-node-vllm-template.yaml
@@ -46,6 +46,7 @@ spec:
metadata:
labels:
app: vllm-canary-${ENV_NAME}
+ sam-canary: "true"
spec:
serviceAccountName: sam-node-sa
nodeSelector:
@@ -107,8 +108,11 @@ spec:
- "--insecure-control-plane"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--bind-addr=127.0.0.1:8080"
+ - "--metrics-addr=0.0.0.0:9090"
ports:
- containerPort: 8080
+ - containerPort: 9090
+ name: metrics
resources:
requests:
cpu: 50m
diff --git a/.github/k8s/sam-router-template.yaml b/.github/k8s/sam-router-template.yaml
index 681a8a8a..3f916732 100644
--- a/.github/k8s/sam-router-template.yaml
+++ b/.github/k8s/sam-router-template.yaml
@@ -55,6 +55,10 @@ spec:
hostPort: 4501
protocol: UDP
name: p2p-udp
+ # Cluster-internal only: no hostPort, and nothing on it is authenticated.
+ - containerPort: 9090
+ protocol: TCP
+ name: metrics
args:
- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
- "--insecure-control-plane"
@@ -63,6 +67,25 @@ spec:
- "--external-addr=/dnsaddr/bootstrap.${ENV_NAME}.sam-mesh.dev"
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--keys-path=/data/router.key"
+ - "--metrics-addr=0.0.0.0:9090"
+ # /healthz answers as soon as the process is up; /readyz once the
+ # router has enrolled and its libp2p host is online.
+ startupProbe:
+ httpGet:
+ path: /healthz
+ port: metrics
+ periodSeconds: 5
+ failureThreshold: 24
+ readinessProbe:
+ httpGet:
+ path: /readyz
+ port: metrics
+ periodSeconds: 10
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: metrics
+ periodSeconds: 20
resources:
requests:
cpu: 100m
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index ee95dbee..d8eac832 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -374,6 +374,12 @@ jobs:
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-router-template.yaml | kubectl apply -f -
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-console-template.yaml | kubectl apply -f -
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID}' < .github/k8s/dns-sync-cronjob-template.yaml | kubectl apply -f -
+ # Managed Prometheus ships with GKE; a cluster without it should not block the mesh rollout.
+ if kubectl get crd podmonitorings.monitoring.googleapis.com >/dev/null 2>&1; then
+ envsubst '${ENV_NAME} ${NAMESPACE}' < .github/k8s/sam-monitoring-template.yaml | kubectl apply -f -
+ else
+ echo "::warning::PodMonitoring CRD not found; managed collection is off on this cluster, metrics will not be scraped"
+ fi
kubectl rollout status deployment/sam-control-plane-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || {
echo "Control Plane Deployment failed!"
diff --git a/tests/e2e/canary_manifests.bats b/tests/e2e/canary_manifests.bats
index d9100138..eb4bad0b 100644
--- a/tests/e2e/canary_manifests.bats
+++ b/tests/e2e/canary_manifests.bats
@@ -61,16 +61,16 @@ render() {
# unservable reports whether every error is this cluster's missing CRDs rather
# than the manifest's fault. The control plane template carries GKE Gateway and
-# HealthCheckPolicy resources, which no kind cluster serves and which nothing
-# here can install, so failing on them would only teach people to ignore this
-# test.
+# HealthCheckPolicy resources, and the monitoring template Managed Prometheus
+# PodMonitorings, which no kind cluster serves and which nothing here can
+# install, so failing on them would only teach people to ignore this test.
#
# The kinds are named rather than matched on "no matches for kind", because a
# Deployment declared against a retired apiVersion fails with that same wording
# -- which made an earlier version of this test pass a manifest whose
# apps/v1beta1 would have broken the rollout it exists to protect.
unservable() {
- local kinds='Gateway|HTTPRoute|HealthCheckPolicy|GCPBackendPolicy'
+ local kinds='Gateway|HTTPRoute|HealthCheckPolicy|GCPBackendPolicy|PodMonitoring'
! grep -qvE "no matches for kind \"(${kinds})\"|ensure CRDs are installed|^[[:space:]]*$" <<<"$1"
}