Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
deploy: scrape the control plane, routers and canaries with Managed Prometheus
Antonio Ojeae9f423456c97
diff --git a/.github/k8s/sam-box-canary-template.yaml b/.github/k8s/sam-box-canary-template.yamlindex 0247d16b..476ab62a 100644--- a/.github/k8s/sam-box-canary-template.yaml+++ b/.github/k8s/sam-box-canary-template.yaml@@ -59,6 +59,7 @@ spec:metadata:labels:app: box-canary-${ENV_NAME}+ sam-canary: "true"spec:serviceAccountName: sam-box-sa# nano-init ships as its own image with no shell in it, so the binary is@@ -84,6 +85,11 @@ spec:# the socket's permissions are the credential.- "--bind-addr="- "--socket-path=/var/run/sam/node.sock"+ # The one TCP port, and it carries nothing an API token would gate.+ - "--metrics-addr=0.0.0.0:9090"+ ports:+ - containerPort: 9090+ name: metricsresources:requests:cpu: 50m@@ -107,6 +113,10 @@ spec:- "--sidecar-socket=/var/run/sam/node.sock"- "--egress-allow=example.com"- "--log-level=debug"+ - "--metrics-addr=0.0.0.0:9091"+ ports:+ - containerPort: 9091+ name: box-metricsresources:requests:cpu: 20mdiff --git a/.github/k8s/sam-monitoring-template.yaml b/.github/k8s/sam-monitoring-template.yamlnew file mode 100644index 00000000..d6dab1cc--- /dev/null+++ b/.github/k8s/sam-monitoring-template.yaml@@ -0,0 +1,53 @@+# Scrape targets for Google Managed Prometheus, which GKE runs by default.+# A PodMonitoring only sees pods in its own namespace, so the canaries get+# their own below.+apiVersion: monitoring.googleapis.com/v1+kind: PodMonitoring+metadata:+ name: sam-control-plane-${ENV_NAME}+ namespace: ${NAMESPACE}+spec:+ selector:+ matchLabels:+ app: sam-control-plane-${ENV_NAME}+ endpoints:+ - port: http+ path: /metrics+ interval: 30s+---+apiVersion: monitoring.googleapis.com/v1+kind: PodMonitoring+metadata:+ name: sam-router-${ENV_NAME}+ namespace: ${NAMESPACE}+spec:+ selector:+ matchLabels:+ app: sam-router-${ENV_NAME}+ endpoints:+ - port: metrics+ path: /metrics+ interval: 30s+---+# Every canary pod carries sam-canary=true; a pod without one of these named+# ports is simply not a target for that endpoint.+apiVersion: monitoring.googleapis.com/v1+kind: PodMonitoring+metadata:+ name: sam-canaries-${ENV_NAME}+ namespace: sam-canary-${ENV_NAME}+spec:+ selector:+ matchLabels:+ sam-canary: "true"+ endpoints:+ - port: metrics+ path: /metrics+ interval: 30s+ - port: box-metrics+ path: /metrics+ interval: 30s+ targetLabels:+ fromPod:+ - from: app+ to: canarydiff --git a/.github/k8s/sam-node-cop-template.yaml b/.github/k8s/sam-node-cop-template.yamlindex 6a2d7474..29357c8b 100644--- a/.github/k8s/sam-node-cop-template.yaml+++ b/.github/k8s/sam-node-cop-template.yaml@@ -30,6 +30,7 @@ spec:metadata:labels:app: cop-canary-${ENV_NAME}+ sam-canary: "true"spec:serviceAccountName: sam-node-sacontainers:@@ -70,8 +71,11 @@ spec:- "--insecure-control-plane"- "--jwt-path=/var/run/secrets/tokens/sam-token"- "--bind-addr=127.0.0.1:8080"+ - "--metrics-addr=0.0.0.0:9090"ports:- containerPort: 8080+ - containerPort: 9090+ name: metricsresources:requests:cpu: 50mdiff --git a/.github/k8s/sam-node-everything-template.yaml b/.github/k8s/sam-node-everything-template.yamlindex b79ac198..b5fa9ae1 100644--- a/.github/k8s/sam-node-everything-template.yaml+++ b/.github/k8s/sam-node-everything-template.yaml@@ -30,6 +30,7 @@ spec:metadata:labels:app: everything-canary-${ENV_NAME}+ sam-canary: "true"spec:serviceAccountName: sam-node-sacontainers:@@ -58,8 +59,11 @@ spec:- "--insecure-control-plane"- "--jwt-path=/var/run/secrets/tokens/sam-token"- "--bind-addr=127.0.0.1:8080"+ - "--metrics-addr=0.0.0.0:9090"ports:- containerPort: 8080+ - containerPort: 9090+ name: metricsresources:requests:cpu: 50mdiff --git a/.github/k8s/sam-node-openclaw-template.yaml b/.github/k8s/sam-node-openclaw-template.yamlindex 64c65976..2ce0a7bf 100644--- a/.github/k8s/sam-node-openclaw-template.yaml+++ b/.github/k8s/sam-node-openclaw-template.yaml@@ -47,6 +47,7 @@ spec:metadata:labels:app: openclaw-canary-${ENV_NAME}+ sam-canary: "true"spec:serviceAccountName: sam-node-sacontainers:@@ -114,8 +115,11 @@ spec:- "--insecure-control-plane"- "--jwt-path=/var/run/secrets/tokens/sam-token"- "--bind-addr=127.0.0.1:8080"+ - "--metrics-addr=0.0.0.0:9090"ports:- containerPort: 8080+ - containerPort: 9090+ name: metricsresources:requests:cpu: 50mdiff --git a/.github/k8s/sam-node-openrouter-template.yaml b/.github/k8s/sam-node-openrouter-template.yamlindex 578166a0..ae22d884 100644--- a/.github/k8s/sam-node-openrouter-template.yaml+++ b/.github/k8s/sam-node-openrouter-template.yaml@@ -32,6 +32,7 @@ spec:metadata:labels:app: openrouter-canary-${ENV_NAME}+ sam-canary: "true"spec:serviceAccountName: sam-node-sacontainers:@@ -67,8 +68,11 @@ spec:- "--insecure-control-plane"- "--jwt-path=/var/run/secrets/tokens/sam-token"- "--bind-addr=127.0.0.1:8080"+ - "--metrics-addr=0.0.0.0:9090"ports:- containerPort: 8080+ - containerPort: 9090+ name: metricsresources:requests:cpu: 500mdiff --git a/.github/k8s/sam-node-template.yaml b/.github/k8s/sam-node-template.yamlindex 7b3bfdad..192fdcd2 100644--- a/.github/k8s/sam-node-template.yaml+++ b/.github/k8s/sam-node-template.yaml@@ -37,6 +37,7 @@ spec:metadata:labels:app: sam-canary-${ENV_NAME}+ sam-canary: "true"spec:serviceAccountName: sam-node-sacontainers:@@ -51,8 +52,12 @@ spec:- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"- "--insecure-control-plane"- "--jwt-path=/var/run/secrets/tokens/sam-token"+ # 9091, not 9090: the agnhost container below owns 9090 in this pod.+ - "--metrics-addr=0.0.0.0:9091"ports:- containerPort: 8080+ - containerPort: 9091+ name: metricsresources:requests:cpu: 50mdiff --git a/.github/k8s/sam-node-vllm-template.yaml b/.github/k8s/sam-node-vllm-template.yamlindex a956e104..0a9dc86e 100644--- a/.github/k8s/sam-node-vllm-template.yaml+++ b/.github/k8s/sam-node-vllm-template.yaml@@ -46,6 +46,7 @@ spec:metadata:labels:app: vllm-canary-${ENV_NAME}+ sam-canary: "true"spec:serviceAccountName: sam-node-sanodeSelector:@@ -107,8 +108,11 @@ spec:- "--insecure-control-plane"- "--jwt-path=/var/run/secrets/tokens/sam-token"- "--bind-addr=127.0.0.1:8080"+ - "--metrics-addr=0.0.0.0:9090"ports:- containerPort: 8080+ - containerPort: 9090+ name: metricsresources:requests:cpu: 50mdiff --git a/.github/k8s/sam-router-template.yaml b/.github/k8s/sam-router-template.yamlindex 681a8a8a..3f916732 100644--- a/.github/k8s/sam-router-template.yaml+++ b/.github/k8s/sam-router-template.yaml@@ -55,6 +55,10 @@ spec:hostPort: 4501protocol: UDPname: p2p-udp+ # Cluster-internal only: no hostPort, and nothing on it is authenticated.+ - containerPort: 9090+ protocol: TCP+ name: metricsargs:- "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"- "--insecure-control-plane"@@ -63,6 +67,25 @@ spec:- "--external-addr=/dnsaddr/bootstrap.${ENV_NAME}.sam-mesh.dev"- "--jwt-path=/var/run/secrets/tokens/sam-token"- "--keys-path=/data/router.key"+ - "--metrics-addr=0.0.0.0:9090"+ # /healthz answers as soon as the process is up; /readyz once the+ # router has enrolled and its libp2p host is online.+ startupProbe:+ httpGet:+ path: /healthz+ port: metrics+ periodSeconds: 5+ failureThreshold: 24+ readinessProbe:+ httpGet:+ path: /readyz+ port: metrics+ periodSeconds: 10+ livenessProbe:+ httpGet:+ path: /healthz+ port: metrics+ periodSeconds: 20resources:requests:cpu: 100mdiff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yamlindex ee95dbee..d8eac832 100644--- a/.github/workflows/deploy.yaml+++ b/.github/workflows/deploy.yaml@@ -374,6 +374,12 @@ jobs:envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-router-template.yaml | kubectl apply -f -envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-console-template.yaml | kubectl apply -f -envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID}' < .github/k8s/dns-sync-cronjob-template.yaml | kubectl apply -f -+ # Managed Prometheus ships with GKE; a cluster without it should not block the mesh rollout.+ if kubectl get crd podmonitorings.monitoring.googleapis.com >/dev/null 2>&1; then+ envsubst '${ENV_NAME} ${NAMESPACE}' < .github/k8s/sam-monitoring-template.yaml | kubectl apply -f -+ else+ echo "::warning::PodMonitoring CRD not found; managed collection is off on this cluster, metrics will not be scraped"+ fikubectl rollout status deployment/sam-control-plane-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || {echo "Control Plane Deployment failed!"diff --git a/tests/e2e/canary_manifests.bats b/tests/e2e/canary_manifests.batsindex d9100138..eb4bad0b 100644--- a/tests/e2e/canary_manifests.bats+++ b/tests/e2e/canary_manifests.bats@@ -61,16 +61,16 @@ render() {# unservable reports whether every error is this cluster's missing CRDs rather# than the manifest's fault. The control plane template carries GKE Gateway and-# HealthCheckPolicy resources, which no kind cluster serves and which nothing-# here can install, so failing on them would only teach people to ignore this-# test.+# HealthCheckPolicy resources, and the monitoring template Managed Prometheus+# PodMonitorings, which no kind cluster serves and which nothing here can+# install, so failing on them would only teach people to ignore this test.## The kinds are named rather than matched on "no matches for kind", because a# Deployment declared against a retired apiVersion fails with that same wording# -- which made an earlier version of this test pass a manifest whose# apps/v1beta1 would have broken the rollout it exists to protect.unservable() {- local kinds='Gateway|HTTPRoute|HealthCheckPolicy|GCPBackendPolicy'+ local kinds='Gateway|HTTPRoute|HealthCheckPolicy|GCPBackendPolicy|PodMonitoring'! grep -qvE "no matches for kind \"(${kinds})\"|ensure CRDs are installed|^[[:space:]]*$" <<<"$1"}