Skip to content

Agent Mesh

ci: check each testnet's health every half hour

Antonio Ojeaa40a9692a76f
diff --git a/.github/workflows/testnet-health.yaml b/.github/workflows/testnet-health.yaml
new file mode 100644
index 00000000..10532b3f
--- /dev/null
+++ b/.github/workflows/testnet-health.yaml
@@ -0,0 +1,181 @@
+name: Testnet Health
+
+# Looks at each public testnet every half hour and says whether a new user
+# could join it right now. deploy.yaml only knows the moment of a rollout;
+# this is what watches in between. A red badge on the README and an open
+# "Testnet <env> is unhealthy" issue are the signals; the issue closes itself
+# on the next green run.
+
+on:
+ schedule:
+ - cron: '*/30 * * * *'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: testnet-health
+ cancel-in-progress: false
+
+jobs:
+ check:
+ name: ${{ matrix.environment }}
+ runs-on: ubuntu-latest
+ # The same GitHub Environments deploy.yaml uses, for the same cluster
+ # credentials and per-testnet variables.
+ environment: ${{ matrix.environment }}
+ strategy:
+ fail-fast: false
+ matrix:
+ environment: [hub, bananas]
+ permissions:
+ contents: read
+ id-token: write
+ issues: write
+ timeout-minutes: 15
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Google Auth
+ uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2
+ with:
+ workload_identity_provider: ${{ vars.WIF_PROVIDER_NAME }}
+ service_account: ${{ vars.SERVICE_ACCOUNT_EMAIL }}
+
+ - name: Set up GKE credentials
+ uses: google-github-actions/get-gke-credentials@3da1e46a907576cefaa90c484278bb5b259dd395 # v3.0.0
+ with:
+ cluster_name: ${{ vars.CLUSTER_NAME }}
+ location: ${{ vars.CLUSTER_REGION }}
+
+ - name: Check the testnet
+ id: check
+ # Every check runs even after one fails, so the report is complete;
+ # the report step turns the outcome into the issue and the exit code.
+ continue-on-error: true
+ env:
+ ENV_NAME: ${{ vars.ENV_NAME }}
+ run: |
+ set -uo pipefail
+ NS="sam-${ENV_NAME}"
+ CNS="sam-canary-${ENV_NAME}"
+ HOST="${ENV_NAME}.sam-mesh.dev"
+ rows=()
+ fails=0
+ ok() { rows+=("| :white_check_mark: | $1 | $2 |"); }
+ bad() { rows+=("| :x: | $1 | $2 |"); fails=$((fails + 1)); }
+
+ # 1. The public enrollment surface, as a new node sees it. /info is
+ # protobuf; router peer ids are ASCII inside it.
+ code=$(curl -s -m 20 -o info.bin -w '%{http_code}' "https://${HOST}/info" || echo 000)
+ routers_public=$(grep -aoE '/p2p/12D3Koo[A-Za-z0-9]+' info.bin 2>/dev/null | sort -u | wc -l)
+ if [[ "${code}" == "200" && "${routers_public}" -ge 1 ]]; then
+ ok "Public \`/info\`" "HTTP 200, ${routers_public} router(s) advertised"
+ else
+ bad "Public \`/info\`" "HTTP ${code}, ${routers_public} router(s) advertised"
+ fi
+
+ # 2. What the control plane knows, read through the API server so no
+ # port is opened to the internet for it.
+ if metrics=$(kubectl get --raw "/api/v1/namespaces/${NS}/services/sam-control-plane-${ENV_NAME}:8080/proxy/metrics" 2>&1); then
+ routers_active=$(awk '$1=="sam_control_plane_routers_active"{print int($2)}' <<<"${metrics}")
+ peers=$(awk '$1=="sam_control_plane_mesh_connected_peers"{print int($2)}' <<<"${metrics}")
+ scrape=$(awk '$1=="sam_control_plane_mesh_state_scrape_success"{print int($2)}' <<<"${metrics}")
+ if [[ "${scrape:-0}" -eq 1 && "${routers_active:-0}" -ge 2 ]]; then
+ ok "Routers holding a lease" "${routers_active}"
+ else
+ bad "Routers holding a lease" "${routers_active:-?} (store scrape success=${scrape:-?}; want >= 2)"
+ fi
+ if [[ "${peers:-0}" -ge 1 ]]; then
+ ok "Peers attached to the mesh" "${peers}"
+ else
+ bad "Peers attached to the mesh" "${peers:-?} (want >= 1)"
+ fi
+ else
+ bad "Control plane metrics" "$(head -c 200 <<<"${metrics}")"
+ fi
+
+ # 3. Routers and canaries are fully rolled out.
+ sts=$(kubectl get statefulset "sam-router-${ENV_NAME}" -n "${NS}" -o jsonpath='{.status.readyReplicas}/{.spec.replicas}' 2>&1 || echo "?/?")
+ if [[ "${sts%%/*}" == "${sts##*/}" && "${sts%%/*}" != "?" && "${sts%%/*}" != "" ]]; then
+ ok "Router StatefulSet" "${sts} ready"
+ else
+ bad "Router StatefulSet" "${sts} ready"
+ fi
+
+ unhealthy=""
+ total=0
+ while read -r name ready want; do
+ [[ -z "${name}" ]] && continue
+ total=$((total + 1))
+ [[ "${ready:-0}" == "${want}" ]] || unhealthy="${unhealthy} ${name}(${ready:-0}/${want})"
+ done < <(kubectl get deployments -n "${CNS}" -o jsonpath='{range .items[*]}{.metadata.name} {.status.availableReplicas} {.spec.replicas}{"\n"}{end}' 2>/dev/null)
+ if [[ "${total}" -gt 0 && -z "${unhealthy}" ]]; then
+ ok "Canaries" "${total} deployments fully available"
+ else
+ bad "Canaries" "${total} deployments; not available:${unhealthy:- (none found)}"
+ fi
+
+ # 4. The cold path: the probe CronJob enrolled a fresh node, found a
+ # service and spoke MCP to it recently. Three schedules of slack.
+ last=$(kubectl get cronjob "sam-probe-${ENV_NAME}" -n "${CNS}" -o jsonpath='{.status.lastSuccessfulTime}' 2>/dev/null || true)
+ if [[ -n "${last}" ]]; then
+ age=$(( $(date +%s) - $(date -d "${last}" +%s) ))
+ if [[ "${age}" -le 2700 ]]; then
+ ok "Cold-path probe" "last success ${age}s ago"
+ else
+ bad "Cold-path probe" "last success ${age}s ago (want <= 2700s)"
+ fi
+ else
+ bad "Cold-path probe" "no successful run recorded"
+ fi
+
+ {
+ echo "## ${ENV_NAME}.sam-mesh.dev"
+ echo
+ echo "| | Check | Result |"
+ echo "|---|---|---|"
+ printf '%s\n' "${rows[@]}"
+ } > report.md
+ cat report.md >> "${GITHUB_STEP_SUMMARY}"
+ cat report.md
+ [[ "${fails}" -eq 0 ]]
+
+ - name: Report
+ if: always()
+ env:
+ GH_TOKEN: ${{ github.token }}
+ ENV_NAME: ${{ vars.ENV_NAME }}
+ OUTCOME: ${{ steps.check.outcome }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ run: |
+ set -euo pipefail
+ title="Testnet ${ENV_NAME} is unhealthy"
+ label="testnet-health"
+ existing=$(gh issue list --state open --label "${label}" --json number,title \
+ --jq ".[] | select(.title == \"${title}\") | .number" | head -n1 || true)
+
+ if [[ "${OUTCOME}" == "success" ]]; then
+ if [[ -n "${existing}" ]]; then
+ gh issue close "${existing}" --comment "Healthy again: ${RUN_URL}"
+ fi
+ exit 0
+ fi
+
+ report=$(cat report.md 2>/dev/null || echo "The check step did not produce a report.")
+ body=$(printf '%s\n\nRun: %s\n_Updated by every failing run; closed automatically by the next healthy one._\n' "${report}" "${RUN_URL}")
+
+ if [[ -n "${existing}" ]]; then
+ # One open issue per testnet, kept current rather than commented on
+ # every half hour.
+ gh issue edit "${existing}" --body "${body}"
+ else
+ gh label create "${label}" --description "Opened by the Testnet Health workflow" --color B60205 2>/dev/null || true
+ gh issue create --title "${title}" --label "${label}" --body "${body}"
+ fi
+ exit 1
diff --git a/README.md b/README.md
index ec634218..7a581bf2 100644
--- a/README.md
+++ b/README.md
@@ -40,6 +40,8 @@ SAM provides the open protocols, cryptographic building blocks, and software to
> **About the Public Developer Testnets:**
> The public endpoints (`bananas.sam-mesh.dev` and `hub.sam-mesh.dev`) are free testbeds created using community resources solely for developer testing, continuous integration, and rapid experimentation. They provide **no guarantees, no uptime commitments, zero SLA, and no sovereign guarantees**. Running on a shared community testnet delegates identity management to the testbed maintainers; true sovereignty requires deploying a dedicated control plane with customer-held keys.
>
+> [![Testnet Health](https://github.com/google/sam/actions/workflows/testnet-health.yaml/badge.svg)](https://github.com/google/sam/actions/workflows/testnet-health.yaml) Every half hour a fresh node's view of each testnet is checked (enrollment surface, routers, canaries, and a cold-path probe that joins and calls a tool); a red badge means one of them is unhealthy and an issue labelled `testnet-health` says which check failed.
+>
> 📖 **Deep Dive:** Read our full **[Digital & Data Sovereignty Architecture](site/content/docs/sovereignty.md)** covering the 5 pillars, fail-closed label gates, uncooperative sandbox confinement, and regulatory alignment (GDPR Chapter V, EU Cloud Sovereignty Framework SEAL-3, EU Data Act).
---