Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
deploy: probe the cold path every 15 minutes and on every rollout
Antonio Ojea448234a4a2dc
diff --git a/.github/k8s/sam-probe-cronjob-template.yaml b/.github/k8s/sam-probe-cronjob-template.yamlnew file mode 100644index 00000000..e4ded877--- /dev/null+++ b/.github/k8s/sam-probe-cronjob-template.yaml@@ -0,0 +1,176 @@+# The cold path, on a schedule: a node with no identity enrolls with the+# control plane, authenticates to a router, finds a service on the mesh and+# speaks MCP to it through the mesh, then exits. The long-lived canaries only+# do this when a pod restarts, so between rollouts nobody re-checks that a new+# user could still join. Each run enrolls a fresh identity; the control+# plane's --node-retention sweep reclaims the rows.+apiVersion: batch/v1+kind: CronJob+metadata:+ name: sam-probe-${ENV_NAME}+ namespace: sam-canary-${ENV_NAME}+spec:+ schedule: "*/15 * * * *"+ concurrencyPolicy: Forbid+ successfulJobsHistoryLimit: 3+ failedJobsHistoryLimit: 5+ jobTemplate:+ spec:+ # One attempt: a retry would hide exactly the flakiness this exists to+ # measure. A failed Job is the signal.+ backoffLimit: 0+ activeDeadlineSeconds: 300+ template:+ metadata:+ labels:+ app: sam-probe-${ENV_NAME}+ spec:+ # Bound to the sam-canary role by deploy.yaml's policy seed, which+ # is what lets the probe call the canaries' services.+ serviceAccountName: sam-node-sa+ restartPolicy: Never+ initContainers:+ # A native sidecar: it stays up for the probe container's lifetime+ # and is stopped when that container exits, so the Job completes.+ - name: sam-node+ restartPolicy: Always+ image: ghcr.io/google/sam-node:${IMAGE_TAG}+ args:+ - "run"+ - "--config=/etc/sam/sam-node.yaml"+ - "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"+ - "--insecure-control-plane"+ - "--jwt-path=/var/run/secrets/tokens/sam-token"+ - "--data-dir=/var/run/sam/data"+ # Socket only: no TCP listener, so no API token to hand out.+ - "--bind-addr="+ - "--socket-path=/var/run/sam/node.sock"+ - "--metrics-addr=127.0.0.1:9090"+ resources:+ requests:+ cpu: 50m+ memory: 64Mi+ limits:+ cpu: 200m+ memory: 256Mi+ volumeMounts:+ - name: config-volume+ mountPath: /etc/sam+ - name: sam-token+ mountPath: /var/run/secrets/tokens+ readOnly: true+ - name: sam-uds+ mountPath: /var/run/sam+ containers:+ - name: probe+ image: alpine/curl:8.12.1+ # The node's socket is 0600 and owned by the uid the node runs as;+ # the probe is that uid rather than root with an override.+ securityContext:+ runAsUser: 65532+ runAsGroup: 65532+ allowPrivilegeEscalation: false+ capabilities:+ drop: ["ALL"]+ env:+ - name: PROBE_SERVICE+ # The everything canary's MCP server (sam-node-everything-template.yaml):+ # a real server, so the node advertises it and initialize succeeds.+ value: everything+ resources:+ requests:+ cpu: 10m+ memory: 16Mi+ limits:+ cpu: 100m+ memory: 64Mi+ volumeMounts:+ - name: sam-uds+ mountPath: /var/run/sam+ command: ["/bin/sh", "-c"]+ args:+ - |+ set -u+ SOCK=/var/run/sam/node.sock+ READYZ=http://127.0.0.1:9090/readyz+ START=$(date +%s)++ # One JSON line per run, for a log-based metric; the exit code is+ # what the Job reports.+ fail() {+ printf '{"probe":"sam-cold-path","ok":false,"stage":"%s","error":"%s","elapsed_s":%d}\n' \+ "$1" "$(printf '%s' "$2" | tr -d '"\n' | cut -c1-300)" "$(( $(date +%s) - START ))"+ exit 1+ }++ # 1. Enroll and authenticate to a router: the node's /readyz.+ until [ "$(curl -s -o /dev/null -w '%{http_code}' "$READYZ")" = "200" ]; do+ [ $(( $(date +%s) - START )) -ge 180 ] && fail enroll "node not ready after 180s"+ sleep 2+ done+ READY_S=$(( $(date +%s) - START ))++ # 2. The router answers a dial.+ CONN=$(curl -sf --unix-socket "$SOCK" http://localhost/debug/connectivity) \+ || fail connectivity "GET /debug/connectivity failed"+ printf '%s' "$CONN" | grep -q '"router_error":false' \+ || fail connectivity "router dial failed: $CONN"+ ROUTER_MS=$(printf '%s' "$CONN" | grep -oE '"router_latency_ms":[0-9]+' | grep -oE '[0-9]+$')+ PEERS=$(printf '%s' "$CONN" | grep -oE '"connected_peers":[0-9]+' | grep -oE '[0-9]+$')++ # 3. A service someone else advertised is discoverable. A node+ # that joined seconds ago has a thin routing table, so the lookup+ # is retried for a while; how long it takes is itself reported.+ PEER=""+ DISCOVER_START=$(date +%s)+ while [ -z "$PEER" ]; do+ if curl -sf --unix-socket "$SOCK" -o /tmp/providers.json \+ "http://localhost/sam/service/discover?type=mcp&name=${PROBE_SERVICE}&timeout=20s"; then+ PEER=$(grep -oE '"peer_id":"[^"]+"' /tmp/providers.json | head -1 | cut -d'"' -f4)+ fi+ [ -n "$PEER" ] && break+ [ $(( $(date +%s) - DISCOVER_START )) -ge 90 ] && fail discover "no provider advertises ${PROBE_SERVICE} after 90s"+ sleep 3+ done+ DISCOVER_S=$(( $(date +%s) - DISCOVER_START ))++ # 4. And answers an MCP initialize through the mesh, end to end.+ # No trailing slash: the path maps onto the service's target_url+ # as-is. The body is JSON or an SSE frame; either names the server.+ set -- $(curl -s --unix-socket "$SOCK" -o /tmp/init.out -w '%{http_code} %{time_total}' \+ -X POST "http://localhost/sam/${PEER}/mcp/${PROBE_SERVICE}" \+ -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' \+ -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"sam-probe","version":"0"}}}')+ CALL_CODE=$1; CALL_S=$2+ [ "$CALL_CODE" = "200" ] || fail call "initialize on ${PEER} returned HTTP ${CALL_CODE}: $(head -c 200 /tmp/init.out)"+ grep -q '"serverInfo"' /tmp/init.out || fail call "initialize on ${PEER} returned no serverInfo: $(head -c 200 /tmp/init.out)"++ printf '{"probe":"sam-cold-path","ok":true,"ready_s":%d,"router_latency_ms":%s,"connected_peers":%s,"discover_s":%d,"call_s":%s,"provider":"%s","elapsed_s":%d}\n' \+ "$READY_S" "${ROUTER_MS:-null}" "${PEERS:-null}" "$DISCOVER_S" "$CALL_S" "$PEER" "$(( $(date +%s) - START ))"+ volumes:+ - name: config-volume+ configMap:+ name: sam-probe-config-${ENV_NAME}+ - name: sam-uds+ emptyDir: {}+ - name: sam-token+ projected:+ sources:+ - serviceAccountToken:+ path: sam-token+ expirationSeconds: 3600+ audience: "sam-control-plane-audience"+---+apiVersion: v1+kind: ConfigMap+metadata:+ name: sam-probe-config-${ENV_NAME}+ namespace: sam-canary-${ENV_NAME}+data:+ sam-node.yaml: |+ version: "v1alpha1"+ attenuation:+ policies: []+ checks: []+ rules: []+ services: []diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yamlindex f9f2eef9..15ead65a 100644--- a/.github/workflows/deploy.yaml+++ b/.github/workflows/deploy.yaml@@ -786,3 +786,49 @@ jobs:exit 1}+ # A fresh node enrolls, reaches a router and calls the everything+ # canary's MCP server through the mesh: what a new user does. The+ # CronJob keeps doing it every 15 minutes; running it once here, after+ # every canary is up, makes the rollout prove it before the deploy is+ # called good.+ - name: Deploy and run the cold-path probe+ env:+ VAR_ENV_NAME: ${{ vars.ENV_NAME }}+ VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }}+ run: |+ export ENV_NAME="${VAR_ENV_NAME}"+ export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"+ export NAMESPACE="sam-${ENV_NAME}"+ export IMAGE_TAG="${VAR_IMAGE_TAG}"+ JOB="sam-probe-rollout-${ENV_NAME}"++ envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-probe-cronjob-template.yaml | kubectl apply -f -++ kubectl delete job "${JOB}" -n "${CANARY_NAMESPACE}" --ignore-not-found+ kubectl create job --from="cronjob/sam-probe-${ENV_NAME}" "${JOB}" -n "${CANARY_NAMESPACE}"++ # `kubectl wait` can only wait for one condition; a failed Job would+ # otherwise sit out the whole timeout.+ for _ in $(seq 1 60); do+ status=$(kubectl get job "${JOB}" -n "${CANARY_NAMESPACE}" -o jsonpath='{range .status.conditions[*]}{.type}={.status}{"\n"}{end}')+ case "${status}" in+ *Complete=True*) break ;;+ *Failed=True*) break ;;+ esac+ sleep 5+ done++ echo "==== probe output ===="+ kubectl logs "job/${JOB}" -n "${CANARY_NAMESPACE}" -c probe --tail=-1 || true++ case "${status:-}" in+ *Complete=True*) echo "Cold-path probe passed." ;;+ *)+ echo "Cold-path probe did not pass: a fresh node could not enroll, reach a router, or call a mesh service."+ echo "==== sam-node sidecar log ===="+ kubectl logs "job/${JOB}" -n "${CANARY_NAMESPACE}" -c sam-node --tail=200 || true+ kubectl describe job "${JOB}" -n "${CANARY_NAMESPACE}" || true+ exit 1+ ;;+ esac+