Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
scale: member ladder, a testnet's capacity in steps of minions
Antonio Ojea9b7652dc0a92
diff --git a/tests/scale/member-ladder-summary.py b/tests/scale/member-ladder-summary.pynew file mode 100755index 00000000..09d30661--- /dev/null+++ b/tests/scale/member-ladder-summary.py@@ -0,0 +1,150 @@+#!/usr/bin/env python3+#+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Render a member-ladder result directory as one table per step.++Usage: member-ladder-summary.py DIR "minion ..." "minion ..." ...++Each step's join reports (one per minion, written when its fleet became+resident) are merged member by member, so the percentiles are exact across+the step rather than an average of averages. The control plane and router+expositions taken around each step give what the mesh did to carry it.+"""+import glob+import json+import os+import re+import sys++out = sys.argv[1]+steps = [s.split() for s in sys.argv[2:]]+++def pct(sorted_values, p):+ if not sorted_values:+ return 0.0+ k = max(0, min(len(sorted_values) - 1, int(round(p / 100 * len(sorted_values) + 0.5)) - 1))+ return sorted_values[k]+++line = re.compile(r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)')+++def parse_all(prefix, tag):+ series = {}+ for path in sorted(glob.glob(os.path.join(out, f"{prefix}-*-{tag}.prom"))):+ pod = os.path.basename(path)[: -len(f"-{tag}.prom")]+ for raw in open(path):+ m = line.match(raw)+ if not m:+ continue+ labels = tuple(sorted(re.findall(r'(\w+)="((?:[^"\\]|\\.)*)"', m.group(2) or ""))) + (("pod", pod),)+ try:+ series[(m.group(1), labels)] = float(m.group(3))+ except ValueError:+ pass+ return series+++def total(series, name, **match):+ return sum(v for (n, labels), v in series.items()+ if n == name and all(dict(labels).get(k) == val for k, val in match.items()))+++def delta_by(before, after, name, label):+ keys = {dict(labels).get(label) for (n, labels) in list(before) + list(after) if n == name}+ return {k: round(total(after, name, **{label: k}) - total(before, name, **{label: k})) for k in sorted(keys, key=str)}+++def p99_of_delta(before, after, name):+ buckets = {}+ for (n, labels), v in after.items():+ if n == name + "_bucket":+ le = dict(labels)["le"]+ buckets[le] = buckets.get(le, 0) + v - before.get((n, labels), 0)+ if not buckets or buckets.get("+Inf", 0) <= 0:+ return None+ target = 0.99 * buckets["+Inf"]+ for le in sorted(buckets, key=lambda s: float(s)):+ if buckets[le] >= target:+ return float(le)+ return float("inf")+++print()+print("| step | minions | resident before | joining | ready | completed | failed | ready p50/p95/max ms | journey p50/p95/max ms | calls/member | elapsed s |")+print("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |")+notes = []+resident = 0+for i, minions in enumerate(steps, 1):+ reports = []+ for m in minions:+ path = os.path.join(out, f"step{i}-{m}.json")+ if os.path.exists(path):+ reports.append(json.load(open(path))["join"])+ if not reports:+ break+ members = [mem for r in reports for mem in r["members"]]+ ready = sorted(mem["ready_ms"] for mem in members if mem.get("ready_ms"))+ journey = sorted(mem["ready_ms"] + mem["first_call_ms"] for mem in members if mem.get("first_call_ms"))+ count = sum(r["count"] for r in reports)+ completed = sum(r["completed"] for r in reports)+ failed = sum(r["failed"] for r in reports)+ attempts = sum(r["call_attempts"] for r in reports)+ elapsed = max(r["elapsed_seconds"] for r in reports)+ print(f"| {i} | {len(reports)}/{len(minions)} | {resident} | {count} | {len(ready)} | {completed} | {failed} | "+ f"{pct(ready,50):.0f}/{pct(ready,95):.0f}/{(ready[-1] if ready else 0):.0f} | "+ f"{pct(journey,50):.0f}/{pct(journey,95):.0f}/{(journey[-1] if journey else 0):.0f} | "+ f"{(attempts / completed if completed else 0):.2f} | {elapsed:.0f} |")+ for r in reports:+ for stage, n in sorted(r.get("errors", {}).items()):+ notes.append(f"step {i}: {n} x {stage}")+ for msg, n in sorted(r.get("call_errors", {}).items()):+ notes.append(f"step {i}: {n} call attempt(s) failed with: {msg}")+ resident += count+for n in notes:+ print(n)++for i in range(1, len(steps) + 1):+ b, a = parse_all("sam-control-plane", f"step{i}-before"), parse_all("sam-control-plane", f"step{i}-after")+ if not (b and a):+ continue+ codes = delta_by(b, a, "sam_control_plane_http_requests_total", "code")+ p99 = p99_of_delta(b, a, "sam_control_plane_http_request_duration_seconds")+ nodes = max([v for (n, labels), v in a.items() if n == "sam_control_plane_enrolled_nodes" and dict(labels).get("role") == "sam:role:node"] or [0])+ peers = max([v for (n, labels), v in a.items() if n == "sam_control_plane_mesh_connected_peers"] or [0])+ print(f"\nstep {i} control plane: requests {codes}; p99 <= {p99} s; enrolled nodes {nodes:.0f}; mesh peers seen {peers:.0f}")+ for r in sorted({re.sub(r"-step\d+-(before|after)\.prom$", "", os.path.basename(p)) for p in glob.glob(os.path.join(out, f"sam-router-*-step{i}-after.prom"))}):+ rb = {k: v for k, v in parse_all("sam-router", f"step{i}-before").items() if dict(k[1]).get("pod") == r}+ ra = {k: v for k, v in parse_all("sam-router", f"step{i}-after").items() if dict(k[1]).get("pod") == r}+ refused = delta_by(rb, ra, "sam_router_inbound_connections_refused_total", "reason")+ hs = delta_by(rb, ra, "sam_router_auth_handshakes_total", "result")+ print(f" {r}: authenticated peers {total(ra, 'sam_router_authenticated_peers'):.0f}, connected {total(ra, 'sam_router_connected_peers'):.0f}, "+ f"rss {total(rb, 'process_resident_memory_bytes')/2**20:.0f} -> {total(ra, 'process_resident_memory_bytes')/2**20:.0f} MiB, "+ f"handshakes {dict((k, v) for k, v in hs.items() if v)}, refused {dict((k, v) for k, v in refused.items() if v) or 'none'}")++# Hold reports, when the fleets were interrupted.+holds = []+for path in sorted(glob.glob(os.path.join(out, "sam-ladder-*.json"))):+ j = json.load(open(path))["join"]+ if j.get("hold"):+ holds.append((os.path.basename(path)[:-5], j["hold"]))+if holds:+ print()+ print("| minion | hold s | resident | fewest ready | flaps | members flapped | outage p50/max ms | not ready at end | exited |")+ print("| --- | --- | --- | --- | --- | --- | --- | --- | --- |")+ for name, h in holds:+ print(f"| {name} | {h['seconds']:.0f} | {h['resident']} | {h['min_ready']} | {h['flaps']} | {h['members_flapped']} | "+ f"{h['outage_ms']['p50']:.0f}/{h['outage_ms']['max']:.0f} | {h['members_not_ready_at_end']} | {h['members_exited']} |")diff --git a/tests/scale/member-ladder.sh b/tests/scale/member-ladder.shnew file mode 100755index 00000000..a37e8cfb--- /dev/null+++ b/tests/scale/member-ladder.sh@@ -0,0 +1,219 @@+#!/usr/bin/env bash+#+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# How many members does a testnet hold?+#+# member-journey.sh asks what the Nth member experiences from one host. This+# climbs: minions join the mesh in steps and every fleet stays resident, so+# each step lands on a mesh already carrying the ones before it. One minion+# carries one address and, by the router's per-address cap, about 500+# members; the ladder is therefore in minions, and a step of two minions is+# a thousand members more.+#+# Each minion runs `sam-bench join` with a long --hold; the join report it+# writes the moment its fleet is resident is pulled back here, so a step is+# judged while its members stay. The control plane and every router are read+# before and after each step, through the API server. At the end, or on+# Ctrl-C, every minion's run is interrupted so its hold report is written,+# and those are pulled back too.+#+# Usage:+# tests/scale/member-ladder.sh --env bananas --per-minion 500 \+# --steps "sam-ladder-01" "sam-ladder-02" "sam-ladder-03 sam-ladder-04 sam-ladder-05" \+# "sam-ladder-06 sam-ladder-07 sam-ladder-08 sam-ladder-09 sam-ladder-10" \+# --out /tmp/ladder+#+# The minions need sam-node and sam-bench on their PATH and a bootstrap token+# at --remote-token-path; the token is minted here with enough usages for the+# whole ladder and revoked on exit, as member-journey.sh does. The minions are+# reached with gcloud over IAP in $ZONE (default us-central1-a) and $PROJECT+# (default: the gcloud configuration's project); $KUBECTL and $GCLOUD may+# carry arguments such as a kubectl context.++set -euo pipefail++ENV_NAME=""+PER_MINION=500+RAMP=20+HOLD=6h+SERVICE=everything+OUT=""+ZONE="${ZONE:-us-central1-a}"+PROJECT="${PROJECT:-}"+REMOTE_TOKEN_PATH=/var/lib/ladder/bootstrap-token+STEPS=()+KUBECTL="${KUBECTL:-kubectl}"+GCLOUD="${GCLOUD:-gcloud}"+SSH_OPTS=(--zone "$ZONE" --tunnel-through-iap --quiet)+if [[ -n "$PROJECT" ]]; then SSH_OPTS+=(--project "$PROJECT"); fi++usage() { sed -n '17,44p' "$0" | sed 's/^# \{0,1\}//'; }++while [[ $# -gt 0 ]]; do+ case "$1" in+ --env) ENV_NAME="$2"; shift 2 ;;+ --per-minion) PER_MINION="$2"; shift 2 ;;+ --ramp) RAMP="$2"; shift 2 ;;+ --hold) HOLD="$2"; shift 2 ;;+ --service) SERVICE="$2"; shift 2 ;;+ --out) OUT="$2"; shift 2 ;;+ --steps) shift; while [[ $# -gt 0 && "$1" != --* ]]; do STEPS+=("$1"); shift; done ;;+ -h|--help) usage; exit 0 ;;+ *) echo "unknown flag: $1" >&2; usage >&2; exit 2 ;;+ esac+done++fail() { echo "member-ladder: $*" >&2; exit 1; }+log() { echo "$(date -u +%FT%TZ) $*"; }+[[ -n "$ENV_NAME" && -n "$OUT" && ${#STEPS[@]} -gt 0 ]] || fail "--env, --out and --steps are required"+NS="sam-${ENV_NAME}"+CONTROL_PLANE="https://${ENV_NAME}.sam-mesh.dev"+mkdir -p "$OUT"+ALL_MINIONS=()+for s in "${STEPS[@]}"; do for m in $s; do ALL_MINIONS+=("$m"); done; done+TOTAL=$(( ${#ALL_MINIONS[@]} * PER_MINION ))++rssh() { # minion cmd...+ local m="$1"; shift+ $GCLOUD compute ssh "$m" "${SSH_OPTS[@]}" --command "$*" 2>/dev/null+}+rscp_to() { $GCLOUD compute scp "${SSH_OPTS[@]}" "$1" "$2:$3" 2>/dev/null; }+rscp_from() { $GCLOUD compute scp "${SSH_OPTS[@]}" "$1:$2" "$3" 2>/dev/null; }++PF_PID=""+TOKEN_ID=""+admin_api() {+ local method="$1" path="$2"; shift 2+ local port+ port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')+ if [[ ! -s "$OUT/admin-header" ]]; then+ $KUBECTL get secret "sam-control-plane-secret-${ENV_NAME}" -n "$NS" -o json \+ | python3 -c 'import json,sys,base64,os; d=json.load(sys.stdin)["data"]["admin-token"]; p=sys.argv[1]; open(p,"w").write("Authorization: Bearer "+base64.b64decode(d).decode()+"\n"); os.chmod(p,0o600)' "$OUT/admin-header"+ fi+ $KUBECTL port-forward "deployment/sam-control-plane-${ENV_NAME}" -n "$NS" "${port}:8080" >/dev/null 2>&1 &+ PF_PID=$!+ for _ in $(seq 1 30); do curl -fsS -o /dev/null "http://127.0.0.1:${port}/healthz" 2>/dev/null && break; sleep 0.5; done+ local status=0+ curl -fsS -X "$method" -H "@${OUT}/admin-header" "$@" "http://127.0.0.1:${port}${path}" || status=$?+ kill "$PF_PID" 2>/dev/null || true; PF_PID=""+ return "$status"+}++snapshot() {+ local tag="$1" pod name+ for pod in $($KUBECTL get pods -n "$NS" -l "app=sam-control-plane-${ENV_NAME}" -o name 2>/dev/null); do+ name="${pod#pod/}"+ $KUBECTL get --raw "/api/v1/namespaces/${NS}/pods/${name}:8080/proxy/metrics" > "$OUT/${name}-${tag}.prom" 2>/dev/null || log "could not read ${name} (${tag})"+ done+ for pod in $($KUBECTL get pods -n "$NS" -l "app=sam-router-${ENV_NAME}" -o name 2>/dev/null); do+ name="${pod#pod/}"+ $KUBECTL get --raw "/api/v1/namespaces/${NS}/pods/${name}:9090/proxy/metrics" > "$OUT/${name}-${tag}.prom" 2>/dev/null || log "could not read ${name} (${tag})"+ done+}++STARTED_MINIONS=()+collect_holds() {+ local m+ for m in "${STARTED_MINIONS[@]}"; do+ rssh "$m" "pkill -INT -x sam-bench || true" || true+ done+ for m in "${STARTED_MINIONS[@]}"; do+ for _ in $(seq 1 60); do+ rssh "$m" "pgrep -x sam-bench >/dev/null" || break+ sleep 2+ done+ rscp_from "$m" "/var/log/ladder/${m}.json" "$OUT/${m}.json" || log "no final report from $m"+ rscp_from "$m" "/var/log/ladder/${m}.log" "$OUT/${m}.log" || true+ done+}++cleanup() {+ local status=$?+ trap - EXIT+ if [[ ${#STARTED_MINIONS[@]} -gt 0 ]]; then+ log "interrupting the resident fleets for their hold reports"+ collect_holds+ snapshot final+ summarize || true+ fi+ [[ -n "$PF_PID" ]] && kill "$PF_PID" 2>/dev/null || true+ if [[ -n "$TOKEN_ID" ]]; then+ admin_api DELETE "/admin/bootstrap-tokens/${TOKEN_ID}" -o /dev/null && log "revoked bootstrap token ${TOKEN_ID}" || log "could not revoke token ${TOKEN_ID}; revoke it by hand"+ fi+ rm -f "$OUT/admin-header" "$OUT/bootstrap-token"+ exit "$status"+}+trap cleanup EXIT INT TERM++summarize() {+ python3 "$(dirname "$0")/member-ladder-summary.py" "$OUT" "${STEPS[@]}"+}++# One token for the whole ladder.+uses=$(( TOTAL + TOTAL / 10 + 20 ))+admin_api POST /admin/bootstrap-tokens -H 'Content-Type: application/json' \+ -d "{\"role\":\"sam:role:node\",\"max_usages\":${uses},\"ttl_hours\":48,\"description\":\"member-ladder ${TOTAL} from ${#ALL_MINIONS[@]} minions\"}" > "$OUT/token.json" || fail "minting the bootstrap token failed"+python3 -c 'import json,sys; t=json.load(open(sys.argv[1])); open(sys.argv[2],"w").write(t["token"])' "$OUT/token.json" "$OUT/bootstrap-token"+TOKEN_ID=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["id"])' "$OUT/token.json")+rm -f "$OUT/token.json"; chmod 600 "$OUT/bootstrap-token"+log "minted bootstrap token ${TOKEN_ID} for ${uses} enrollments"++log "placing the token on ${#ALL_MINIONS[@]} minions"+for m in "${ALL_MINIONS[@]}"; do+ rssh "$m" "test -x /usr/local/bin/sam-bench && test -x /usr/local/bin/sam-node" || fail "$m has no binaries yet (startup script still running?)"+ rscp_to "$OUT/bootstrap-token" "$m" /tmp/bootstrap-token+ rssh "$m" "sudo install -m 0644 /tmp/bootstrap-token ${REMOTE_TOKEN_PATH} && rm -f /tmp/bootstrap-token" || fail "could not place the token on $m"+done++: > "$OUT/events.jsonl"+event() { printf '{"event":"%s","at":"%s"}\n' "$1" "$(date -u +%FT%TZ)" >> "$OUT/events.jsonl"; }++resident=0+step_no=0+for step in "${STEPS[@]}"; do+ step_no=$((step_no + 1))+ read -r -a minions <<< "$step"+ target=$(( resident + ${#minions[@]} * PER_MINION ))+ log "step ${step_no}: ${#minions[@]} minion(s) x ${PER_MINION} join a mesh holding ${resident} -> ${target}"+ snapshot "step${step_no}-before"+ event "step${step_no}-start"+ for m in "${minions[@]}"; do+ # The run detaches from the ssh session; its stderr is the progress log.+ rssh "$m" "ulimit -n 1048576; rm -rf /var/lib/ladder/${m} /var/log/ladder/${m}.json /var/log/ladder/resident; nohup sam-bench join --node-bin /usr/local/bin/sam-node --control-plane ${CONTROL_PLANE} --bootstrap-token-path ${REMOTE_TOKEN_PATH} --service ${SERVICE} --count ${PER_MINION} --ramp ${RAMP} --hold ${HOLD} --dir /var/lib/ladder/${m} --label minion=${m} --label step=${step_no} --label resident_before=${resident} --resident-marker /var/log/ladder/resident --out /var/log/ladder/${m}.json > /var/log/ladder/${m}.log 2>&1 < /dev/null &" || fail "could not start the run on $m"+ STARTED_MINIONS+=("$m")+ done+ # Resident means every member's journey has ended, one way or the other.+ deadline=$(( $(date +%s) + 900 ))+ for m in "${minions[@]}"; do+ until rssh "$m" "test -f /var/log/ladder/resident"; do+ [[ $(date +%s) -lt $deadline ]] || fail "$m was not resident within 15 minutes; see /var/log/ladder/${m}.log on it"+ rssh "$m" "pgrep -x sam-bench >/dev/null" || fail "the run on $m ended before its fleet was resident"+ sleep 10+ done+ rscp_from "$m" "/var/log/ladder/${m}.json" "$OUT/step${step_no}-${m}.json" || fail "could not fetch the join report from $m"+ done+ event "step${step_no}-resident"+ # Let the mesh settle on the new population before it is read.+ sleep 60+ snapshot "step${step_no}-after"+ resident=$target+ summarize || true+ log "step ${step_no} done: ${resident} members resident"+done++log "ladder complete: ${resident} members resident; holding until Ctrl-C or ${HOLD}"+# Stay until the first minion's hold ends or the operator interrupts.+while rssh "${STARTED_MINIONS[0]}" "pgrep -x sam-bench >/dev/null"; do sleep 60; done