Skip to content

Agent Mesh

deploy: a resident soak fleet on each testnet, and an alert when a member loses the mesh

Antonio Ojea4b39056a6827
diff --git a/.github/k8s/sam-monitoring-template.yaml b/.github/k8s/sam-monitoring-template.yaml
index 140d441c..d627908a 100644
--- a/.github/k8s/sam-monitoring-template.yaml
+++ b/.github/k8s/sam-monitoring-template.yaml
@@ -153,3 +153,16 @@ spec:
annotations:
summary: "Canary {{ $labels.pod }} on ${ENV_NAME} has not been ready for 20m"
description: "`kubectl logs -n sam-canary-${ENV_NAME} {{ $labels.pod }} -c caller | tail -3` gives the error and the per-router detail of the last calls."
+ # A member's own account of the mesh: its node holds no authenticated
+ # router connection. A router rollout costs a member seconds; five
+ # minutes is a member that cannot get back, which after a key rotation
+ # or a refresh is how a fleet falls off the mesh one pod at a time. The
+ # soak fleet (sam-soak-template.yaml) exists to make this fire first.
+ - alert: SamCanaryMemberDisconnected
+ expr: min by (pod) (sam_node_mesh_connected{namespace="sam-canary-${ENV_NAME}"}) == 0
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: "Member {{ $labels.pod }} on ${ENV_NAME} has had no router for 5m"
+ description: "`kubectl logs -n sam-canary-${ENV_NAME} {{ $labels.pod }} -c sam-node | grep -iE 'router|auth|renew' | tail -5` says whether it cannot dial, is refused at the handshake, or holds a credential that no longer verifies; compare with sam_router_auth_handshakes_total and sam_router_inbound_connections_refused_total on the routers."
diff --git a/.github/k8s/sam-soak-template.yaml b/.github/k8s/sam-soak-template.yaml
new file mode 100644
index 00000000..11dc9bea
--- /dev/null
+++ b/.github/k8s/sam-soak-template.yaml
@@ -0,0 +1,108 @@
+# A resident fleet: members that join the mesh and stay, publishing nothing
+# and calling nothing. The other canaries are two pods each and restart when
+# their own service misbehaves, so between rollouts nobody watches what a
+# plain member experiences over days: a router rollout under it, the daily
+# key rotation, its credential refresh. These do. Readiness is the node's
+# /readyz, which is 200 only while it holds an authenticated router
+# connection, so a member that lost the mesh is a pod that is not ready and
+# the canary alerts see it. There is deliberately no liveness probe: a
+# restart would hide exactly the reconnection this fleet exists to observe.
+#
+# Every pod carries sam-canary=true, so the canary PodMonitoring scrapes
+# sam_node_mesh_connected from each; sam-monitoring-template.yaml alerts on
+# a member that reads 0 for five minutes.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sam-soak-config-${ENV_NAME}
+ namespace: sam-canary-${ENV_NAME}
+data:
+ sam-node.yaml: |
+ version: "v1alpha1"
+ attenuation:
+ policies: []
+ checks: []
+ rules: []
+ services: []
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: sam-soak-${ENV_NAME}
+ namespace: sam-canary-${ENV_NAME}
+spec:
+ replicas: 20
+ # A rollout of this fleet is itself a burst of joiners; let it be one
+ # rather than a trickle.
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 100%
+ maxUnavailable: 0
+ selector:
+ matchLabels:
+ app: sam-soak-${ENV_NAME}
+ template:
+ metadata:
+ labels:
+ app: sam-soak-${ENV_NAME}
+ sam-canary: "true"
+ spec:
+ # Bound to the sam-canary role by deploy.yaml's policy seed.
+ serviceAccountName: sam-node-sa
+ terminationGracePeriodSeconds: 10
+ containers:
+ - name: sam-node
+ image: ghcr.io/google/sam-node:${IMAGE_TAG}
+ args:
+ - "run"
+ - "--config=/etc/sam/sam-node.yaml"
+ - "--control-plane=http://sam-control-plane-${ENV_NAME}.${NAMESPACE}.svc.cluster.local:8080"
+ - "--insecure-control-plane"
+ - "--jwt-path=/var/run/secrets/tokens/sam-token"
+ - "--data-dir=/var/run/sam/data"
+ # Socket only: no TCP listener, so no API token to hand out.
+ - "--bind-addr="
+ - "--socket-path=/var/run/sam/node.sock"
+ - "--metrics-addr=0.0.0.0:9090"
+ ports:
+ - containerPort: 9090
+ name: metrics
+ readinessProbe:
+ httpGet:
+ path: /readyz
+ port: metrics
+ periodSeconds: 10
+ failureThreshold: 3
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: ["ALL"]
+ resources:
+ requests:
+ cpu: 20m
+ memory: 64Mi
+ limits:
+ cpu: 200m
+ memory: 256Mi
+ volumeMounts:
+ - name: config-volume
+ mountPath: /etc/sam
+ - name: sam-token
+ mountPath: /var/run/secrets/tokens
+ readOnly: true
+ - name: sam-uds
+ mountPath: /var/run/sam
+ volumes:
+ - name: config-volume
+ configMap:
+ name: sam-soak-config-${ENV_NAME}
+ - name: sam-uds
+ emptyDir: {}
+ - name: sam-token
+ projected:
+ sources:
+ - serviceAccountToken:
+ path: sam-token
+ expirationSeconds: 3600
+ audience: "sam-control-plane-audience"
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index cc571e7a..eee466a4 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -731,6 +731,32 @@ jobs:
exit 1
}
+ # A resident fleet of plain members that join and stay. Its readiness
+ # is each node's /readyz, so this rollout only completes once every
+ # member holds a router, and every later deploy restarts the routers
+ # under it.
+ - name: Deploy Soak Fleet
+ env:
+ VAR_ENV_NAME: ${{ vars.ENV_NAME }}
+ VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }}
+ run: |
+ export ENV_NAME="${VAR_ENV_NAME}"
+ export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"
+ export NAMESPACE="sam-${ENV_NAME}"
+ export IMAGE_TAG="${VAR_IMAGE_TAG}"
+
+ envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-soak-template.yaml | kubectl apply -f -
+ kubectl rollout status deployment/sam-soak-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=300s || {
+ echo "Soak fleet rollout failed: a member could not join or hold a router."
+ kubectl describe deployment/sam-soak-${ENV_NAME} -n ${CANARY_NAMESPACE} || true
+ kubectl get pods -n ${CANARY_NAMESPACE} -l app=sam-soak-${ENV_NAME} -o wide || true
+ for pod in $(kubectl get pods -n ${CANARY_NAMESPACE} -l app=sam-soak-${ENV_NAME} -o name | head -3); do
+ echo "==== ${pod} ===="
+ kubectl logs -n ${CANARY_NAMESPACE} "${pod#pod/}" --tail=40 || true
+ done
+ exit 1
+ }
+
# Two long-lived pairs with no sam-node: an agent written with one SDK
# and the official A2A SDK, and a caller written with the other that
# sends it a message with the A2A SDK's client every few minutes, by