Skip to content

Agent Mesh

deploy: probe the cold path every 15 minutes and on every rollout

Antonio Ojea448234a4a2dc
diff --git a/.github/k8s/sam-probe-cronjob-template.yaml b/.github/k8s/sam-probe-cronjob-template.yaml
new file mode 100644
index 00000000..e4ded877
--- /dev/null
+++ b/.github/k8s/sam-probe-cronjob-template.yaml
@@ -0,0 +1,176 @@
+# The cold path, on a schedule: a node with no identity enrolls with the
+# control plane, authenticates to a router, finds a service on the mesh and
+# speaks MCP to it through the mesh, then exits. The long-lived canaries only
+# do this when a pod restarts, so between rollouts nobody re-checks that a new
+# user could still join. Each run enrolls a fresh identity; the control
+# plane's --node-retention sweep reclaims the rows.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: sam-probe-${ENV_NAME}
+ namespace: sam-canary-${ENV_NAME}
+spec:
+ schedule: "*/15 * * * *"
+ concurrencyPolicy: Forbid
+ successfulJobsHistoryLimit: 3
+ failedJobsHistoryLimit: 5
+ jobTemplate:
+ spec:
+ # One attempt: a retry would hide exactly the flakiness this exists to
+ # measure. A failed Job is the signal.
+ backoffLimit: 0
+ activeDeadlineSeconds: 300
+ template:
+ metadata:
+ labels:
+ app: sam-probe-${ENV_NAME}
+ spec:
+ # Bound to the sam-canary role by deploy.yaml's policy seed, which
+ # is what lets the probe call the canaries' services.
+ serviceAccountName: sam-node-sa
+ restartPolicy: Never
+ initContainers:
+ # A native sidecar: it stays up for the probe container's lifetime
+ # and is stopped when that container exits, so the Job completes.
+ - name: sam-node
+ restartPolicy: Always
+ image: ghcr.io/google/sam-node:${IMAGE_TAG}
+ args:
+ - "run"
+ - "--config=/etc/sam/sam-node.yaml"
+ - "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
+ - "--jwt-path=/var/run/secrets/tokens/sam-token"
+ - "--data-dir=/var/run/sam/data"
+ # Socket only: no TCP listener, so no API token to hand out.
+ - "--bind-addr="
+ - "--socket-path=/var/run/sam/node.sock"
+ - "--metrics-addr=127.0.0.1:9090"
+ resources:
+ requests:
+ cpu: 50m
+ memory: 64Mi
+ limits:
+ cpu: 200m
+ memory: 256Mi
+ volumeMounts:
+ - name: config-volume
+ mountPath: /etc/sam
+ - name: sam-token
+ mountPath: /var/run/secrets/tokens
+ readOnly: true
+ - name: sam-uds
+ mountPath: /var/run/sam
+ containers:
+ - name: probe
+ image: alpine/curl:8.12.1
+ # The node's socket is 0600 and owned by the uid the node runs as;
+ # the probe is that uid rather than root with an override.
+ securityContext:
+ runAsUser: 65532
+ runAsGroup: 65532
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: ["ALL"]
+ env:
+ - name: PROBE_SERVICE
+ # The everything canary's MCP server (sam-node-everything-template.yaml):
+ # a real server, so the node advertises it and initialize succeeds.
+ value: everything
+ resources:
+ requests:
+ cpu: 10m
+ memory: 16Mi
+ limits:
+ cpu: 100m
+ memory: 64Mi
+ volumeMounts:
+ - name: sam-uds
+ mountPath: /var/run/sam
+ command: ["/bin/sh", "-c"]
+ args:
+ - |
+ set -u
+ SOCK=/var/run/sam/node.sock
+ READYZ=http://127.0.0.1:9090/readyz
+ START=$(date +%s)
+
+ # One JSON line per run, for a log-based metric; the exit code is
+ # what the Job reports.
+ fail() {
+ printf '{"probe":"sam-cold-path","ok":false,"stage":"%s","error":"%s","elapsed_s":%d}\n' \
+ "$1" "$(printf '%s' "$2" | tr -d '"\n' | cut -c1-300)" "$(( $(date +%s) - START ))"
+ exit 1
+ }
+
+ # 1. Enroll and authenticate to a router: the node's /readyz.
+ until [ "$(curl -s -o /dev/null -w '%{http_code}' "$READYZ")" = "200" ]; do
+ [ $(( $(date +%s) - START )) -ge 180 ] && fail enroll "node not ready after 180s"
+ sleep 2
+ done
+ READY_S=$(( $(date +%s) - START ))
+
+ # 2. The router answers a dial.
+ CONN=$(curl -sf --unix-socket "$SOCK" http://localhost/debug/connectivity) \
+ || fail connectivity "GET /debug/connectivity failed"
+ printf '%s' "$CONN" | grep -q '"router_error":false' \
+ || fail connectivity "router dial failed: $CONN"
+ ROUTER_MS=$(printf '%s' "$CONN" | grep -oE '"router_latency_ms":[0-9]+' | grep -oE '[0-9]+$')
+ PEERS=$(printf '%s' "$CONN" | grep -oE '"connected_peers":[0-9]+' | grep -oE '[0-9]+$')
+
+ # 3. A service someone else advertised is discoverable. A node
+ # that joined seconds ago has a thin routing table, so the lookup
+ # is retried for a while; how long it takes is itself reported.
+ PEER=""
+ DISCOVER_START=$(date +%s)
+ while [ -z "$PEER" ]; do
+ if curl -sf --unix-socket "$SOCK" -o /tmp/providers.json \
+ "http://localhost/sam/service/discover?type=mcp&name=${PROBE_SERVICE}&timeout=20s"; then
+ PEER=$(grep -oE '"peer_id":"[^"]+"' /tmp/providers.json | head -1 | cut -d'"' -f4)
+ fi
+ [ -n "$PEER" ] && break
+ [ $(( $(date +%s) - DISCOVER_START )) -ge 90 ] && fail discover "no provider advertises ${PROBE_SERVICE} after 90s"
+ sleep 3
+ done
+ DISCOVER_S=$(( $(date +%s) - DISCOVER_START ))
+
+ # 4. And answers an MCP initialize through the mesh, end to end.
+ # No trailing slash: the path maps onto the service's target_url
+ # as-is. The body is JSON or an SSE frame; either names the server.
+ set -- $(curl -s --unix-socket "$SOCK" -o /tmp/init.out -w '%{http_code} %{time_total}' \
+ -X POST "http://localhost/sam/${PEER}/mcp/${PROBE_SERVICE}" \
+ -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' \
+ -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"sam-probe","version":"0"}}}')
+ CALL_CODE=$1; CALL_S=$2
+ [ "$CALL_CODE" = "200" ] || fail call "initialize on ${PEER} returned HTTP ${CALL_CODE}: $(head -c 200 /tmp/init.out)"
+ grep -q '"serverInfo"' /tmp/init.out || fail call "initialize on ${PEER} returned no serverInfo: $(head -c 200 /tmp/init.out)"
+
+ printf '{"probe":"sam-cold-path","ok":true,"ready_s":%d,"router_latency_ms":%s,"connected_peers":%s,"discover_s":%d,"call_s":%s,"provider":"%s","elapsed_s":%d}\n' \
+ "$READY_S" "${ROUTER_MS:-null}" "${PEERS:-null}" "$DISCOVER_S" "$CALL_S" "$PEER" "$(( $(date +%s) - START ))"
+ volumes:
+ - name: config-volume
+ configMap:
+ name: sam-probe-config-${ENV_NAME}
+ - name: sam-uds
+ emptyDir: {}
+ - name: sam-token
+ projected:
+ sources:
+ - serviceAccountToken:
+ path: sam-token
+ expirationSeconds: 3600
+ audience: "sam-control-plane-audience"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sam-probe-config-${ENV_NAME}
+ namespace: sam-canary-${ENV_NAME}
+data:
+ sam-node.yaml: |
+ version: "v1alpha1"
+ attenuation:
+ policies: []
+ checks: []
+ rules: []
+ services: []
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index f9f2eef9..15ead65a 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -786,3 +786,49 @@ jobs:
exit 1
}
+ # A fresh node enrolls, reaches a router and calls the everything
+ # canary's MCP server through the mesh: what a new user does. The
+ # CronJob keeps doing it every 15 minutes; running it once here, after
+ # every canary is up, makes the rollout prove it before the deploy is
+ # called good.
+ - name: Deploy and run the cold-path probe
+ env:
+ VAR_ENV_NAME: ${{ vars.ENV_NAME }}
+ VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }}
+ run: |
+ export ENV_NAME="${VAR_ENV_NAME}"
+ export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"
+ export NAMESPACE="sam-${ENV_NAME}"
+ export IMAGE_TAG="${VAR_IMAGE_TAG}"
+ JOB="sam-probe-rollout-${ENV_NAME}"
+
+ envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-probe-cronjob-template.yaml | kubectl apply -f -
+
+ kubectl delete job "${JOB}" -n "${CANARY_NAMESPACE}" --ignore-not-found
+ kubectl create job --from="cronjob/sam-probe-${ENV_NAME}" "${JOB}" -n "${CANARY_NAMESPACE}"
+
+ # `kubectl wait` can only wait for one condition; a failed Job would
+ # otherwise sit out the whole timeout.
+ for _ in $(seq 1 60); do
+ status=$(kubectl get job "${JOB}" -n "${CANARY_NAMESPACE}" -o jsonpath='{range .status.conditions[*]}{.type}={.status}{"\n"}{end}')
+ case "${status}" in
+ *Complete=True*) break ;;
+ *Failed=True*) break ;;
+ esac
+ sleep 5
+ done
+
+ echo "==== probe output ===="
+ kubectl logs "job/${JOB}" -n "${CANARY_NAMESPACE}" -c probe --tail=-1 || true
+
+ case "${status:-}" in
+ *Complete=True*) echo "Cold-path probe passed." ;;
+ *)
+ echo "Cold-path probe did not pass: a fresh node could not enroll, reach a router, or call a mesh service."
+ echo "==== sam-node sidecar log ===="
+ kubectl logs "job/${JOB}" -n "${CANARY_NAMESPACE}" -c sam-node --tail=200 || true
+ kubectl describe job "${JOB}" -n "${CANARY_NAMESPACE}" || true
+ exit 1
+ ;;
+ esac
+