Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
deploy: a resident soak fleet on each testnet, and an alert when a member loses the mesh
Antonio Ojea4b39056a6827
diff --git a/.github/k8s/sam-monitoring-template.yaml b/.github/k8s/sam-monitoring-template.yamlindex 140d441c..d627908a 100644--- a/.github/k8s/sam-monitoring-template.yaml+++ b/.github/k8s/sam-monitoring-template.yaml@@ -153,3 +153,16 @@ spec:annotations:summary: "Canary {{ $labels.pod }} on ${ENV_NAME} has not been ready for 20m"description: "`kubectl logs -n sam-canary-${ENV_NAME} {{ $labels.pod }} -c caller | tail -3` gives the error and the per-router detail of the last calls."+ # A member's own account of the mesh: its node holds no authenticated+ # router connection. A router rollout costs a member seconds; five+ # minutes is a member that cannot get back, which after a key rotation+ # or a refresh is how a fleet falls off the mesh one pod at a time. The+ # soak fleet (sam-soak-template.yaml) exists to make this fire first.+ - alert: SamCanaryMemberDisconnected+ expr: min by (pod) (sam_node_mesh_connected{namespace="sam-canary-${ENV_NAME}"}) == 0+ for: 5m+ labels:+ severity: warning+ annotations:+ summary: "Member {{ $labels.pod }} on ${ENV_NAME} has had no router for 5m"+ description: "`kubectl logs -n sam-canary-${ENV_NAME} {{ $labels.pod }} -c sam-node | grep -iE 'router|auth|renew' | tail -5` says whether it cannot dial, is refused at the handshake, or holds a credential that no longer verifies; compare with sam_router_auth_handshakes_total and sam_router_inbound_connections_refused_total on the routers."diff --git a/.github/k8s/sam-soak-template.yaml b/.github/k8s/sam-soak-template.yamlnew file mode 100644index 00000000..11dc9bea--- /dev/null+++ b/.github/k8s/sam-soak-template.yaml@@ -0,0 +1,108 @@+# A resident fleet: members that join the mesh and stay, publishing nothing+# and calling nothing. The other canaries are two pods each and restart when+# their own service misbehaves, so between rollouts nobody watches what a+# plain member experiences over days: a router rollout under it, the daily+# key rotation, its credential refresh. These do. Readiness is the node's+# /readyz, which is 200 only while it holds an authenticated router+# connection, so a member that lost the mesh is a pod that is not ready and+# the canary alerts see it. There is deliberately no liveness probe: a+# restart would hide exactly the reconnection this fleet exists to observe.+#+# Every pod carries sam-canary=true, so the canary PodMonitoring scrapes+# sam_node_mesh_connected from each; sam-monitoring-template.yaml alerts on+# a member that reads 0 for five minutes.+apiVersion: v1+kind: ConfigMap+metadata:+ name: sam-soak-config-${ENV_NAME}+ namespace: sam-canary-${ENV_NAME}+data:+ sam-node.yaml: |+ version: "v1alpha1"+ attenuation:+ policies: []+ checks: []+ rules: []+ services: []+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: sam-soak-${ENV_NAME}+ namespace: sam-canary-${ENV_NAME}+spec:+ replicas: 20+ # A rollout of this fleet is itself a burst of joiners; let it be one+ # rather than a trickle.+ strategy:+ type: RollingUpdate+ rollingUpdate:+ maxSurge: 100%+ maxUnavailable: 0+ selector:+ matchLabels:+ app: sam-soak-${ENV_NAME}+ template:+ metadata:+ labels:+ app: sam-soak-${ENV_NAME}+ sam-canary: "true"+ spec:+ # Bound to the sam-canary role by deploy.yaml's policy seed.+ serviceAccountName: sam-node-sa+ terminationGracePeriodSeconds: 10+ containers:+ - name: sam-node+ image: ghcr.io/google/sam-node:${IMAGE_TAG}+ args:+ - "run"+ - "--config=/etc/sam/sam-node.yaml"+ - "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"+ - "--insecure-control-plane"+ - "--jwt-path=/var/run/secrets/tokens/sam-token"+ - "--data-dir=/var/run/sam/data"+ # Socket only: no TCP listener, so no API token to hand out.+ - "--bind-addr="+ - "--socket-path=/var/run/sam/node.sock"+ - "--metrics-addr=0.0.0.0:9090"+ ports:+ - containerPort: 9090+ name: metrics+ readinessProbe:+ httpGet:+ path: /readyz+ port: metrics+ periodSeconds: 10+ failureThreshold: 3+ securityContext:+ allowPrivilegeEscalation: false+ capabilities:+ drop: ["ALL"]+ resources:+ requests:+ cpu: 20m+ memory: 64Mi+ limits:+ cpu: 200m+ memory: 256Mi+ volumeMounts:+ - name: config-volume+ mountPath: /etc/sam+ - name: sam-token+ mountPath: /var/run/secrets/tokens+ readOnly: true+ - name: sam-uds+ mountPath: /var/run/sam+ volumes:+ - name: config-volume+ configMap:+ name: sam-soak-config-${ENV_NAME}+ - name: sam-uds+ emptyDir: {}+ - name: sam-token+ projected:+ sources:+ - serviceAccountToken:+ path: sam-token+ expirationSeconds: 3600+ audience: "sam-control-plane-audience"diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yamlindex cc571e7a..eee466a4 100644--- a/.github/workflows/deploy.yaml+++ b/.github/workflows/deploy.yaml@@ -731,6 +731,32 @@ jobs:exit 1}+ # A resident fleet of plain members that join and stay. Its readiness+ # is each node's /readyz, so this rollout only completes once every+ # member holds a router, and every later deploy restarts the routers+ # under it.+ - name: Deploy Soak Fleet+ env:+ VAR_ENV_NAME: ${{ vars.ENV_NAME }}+ VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }}+ run: |+ export ENV_NAME="${VAR_ENV_NAME}"+ export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"+ export NAMESPACE="sam-${ENV_NAME}"+ export IMAGE_TAG="${VAR_IMAGE_TAG}"++ envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-soak-template.yaml | kubectl apply -f -+ kubectl rollout status deployment/sam-soak-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=300s || {+ echo "Soak fleet rollout failed: a member could not join or hold a router."+ kubectl describe deployment/sam-soak-${ENV_NAME} -n ${CANARY_NAMESPACE} || true+ kubectl get pods -n ${CANARY_NAMESPACE} -l app=sam-soak-${ENV_NAME} -o wide || true+ for pod in $(kubectl get pods -n ${CANARY_NAMESPACE} -l app=sam-soak-${ENV_NAME} -o name | head -3); do+ echo "==== ${pod} ===="+ kubectl logs -n ${CANARY_NAMESPACE} "${pod#pod/}" --tail=40 || true+ done+ exit 1+ }+# Two long-lived pairs with no sam-node: an agent written with one SDK# and the official A2A SDK, and a caller written with the other that# sends it a message with the A2A SDK's client every few minutes, by