Skip to content

Agent Mesh

scale: collect what a fleet run has to report

Antonio Ojead7eff6f79021
diff --git a/scripts/launch-microvms.sh b/scripts/launch-microvms.sh
index cbe5b829..d926a5fe 100755
--- a/scripts/launch-microvms.sh
+++ b/scripts/launch-microvms.sh
@@ -86,6 +86,13 @@ fc_put() {
"http://localhost$2" -H 'Content-Type: application/json' -d "$3" > /dev/null
}
+# When each sandbox was launched, so a run can say how long a population took
+# to come up rather than only that it did. Readiness is not recorded here: a
+# sandbox is useful when its agent reaches the mesh, and only the node knows
+# that. Pair this with the agent count over time from collect-fleet.sh.
+TIMINGS="${LOG_DIR}/launch-timings.csv"
+echo "index,launched_epoch_ms" > "${TIMINGS}"
+
echo "=== Launching $COUNT agent sandboxes ==="
for i in $(seq 1 "$COUNT"); do
VM_ID="vm-$i"
@@ -159,6 +166,7 @@ EOF
fc_put "$API_SOCKET" /actions '{ "action_type": "InstanceStart" }'
+ echo "$i,$(date +%s%3N)" >> "${TIMINGS}"
echo "Started $VM_ID as agent-${i}.${AGENT_DOMAIN} (${VM_VCPUS} vCPU, ${VM_MEM_MIB} MiB)"
done
diff --git a/tests/scale/collect-fleet.sh b/tests/scale/collect-fleet.sh
new file mode 100755
index 00000000..ace65d26
--- /dev/null
+++ b/tests/scale/collect-fleet.sh
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Watch a fleet of agent sandboxes come up.
+#
+# The headline number for a run like this is how many agents the mesh is
+# actually serving, and it is not the number of microVMs somebody started: a
+# guest that booted and never reached the boundary is not an agent, it is a
+# process. So this asks the node, which is the only party that knows, and
+# samples it over time rather than once at the end -- the shape of the curve is
+# what says whether a population came up smoothly or fell over partway.
+#
+# Alongside it, what the fleet costs the host. Resident memory is summed per
+# process class rather than taken from the machine total, so a run can say what
+# an agent cost rather than what was left over.
+#
+# Usage:
+# tests/scale/collect-fleet.sh --node-socket /var/run/sam-node.sock \
+# --duration 300 --out results.jsonl
+
+set -euo pipefail
+
+NODE_SOCKET="/var/run/sam-node.sock"
+DURATION=300
+INTERVAL=5
+OUT=""
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --node-socket) NODE_SOCKET="$2"; shift 2 ;;
+ --duration) DURATION="$2"; shift 2 ;;
+ --interval) INTERVAL="$2"; shift 2 ;;
+ --out) OUT="$2"; shift 2 ;;
+ -h|--help) sed -n '17,32p' "$0"; exit 0 ;;
+ *) echo "unknown flag: $1" >&2; exit 2 ;;
+ esac
+done
+
+OUT="${OUT:-fleet-$(date +%Y%m%dT%H%M%S).jsonl}"
+
+# Resident, not configured. A guest's mem_size_mib is a ceiling; what the host
+# pays is the pages it touched, and that is what density is built from.
+rss_total() {
+ local pattern="$1" total=0 pid
+ for pid in $(pgrep -f "${pattern}" 2>/dev/null || true); do
+ local kb
+ kb="$(awk '/VmRSS/ {print $2}' "/proc/${pid}/status" 2>/dev/null || echo 0)"
+ total=$((total + ${kb:-0}))
+ done
+ echo "${total}"
+}
+
+# pgrep -c prints 0 AND exits non-zero when nothing matches, so a `|| echo 0`
+# fallback emits it twice and quietly corrupts the JSON.
+count_of() {
+ local n
+ n="$(pgrep -cf "$1" 2>/dev/null)"
+ echo "${n:-0}"
+}
+
+# Nothing below may abort the run: a sample that could not be taken is a zero,
+# not a reason to stop watching a fleet that took minutes to start.
+set +e
+
+# The node counts distinct agents it has served for a local gateway, which is
+# the closest thing to "agents in the mesh" that anything can honestly report.
+#
+# The curl is guarded because a scrape that fails must produce a zero sample
+# rather than end the collection: under pipefail a failed curl fails the whole
+# pipeline, and a collector that dies the first time the node is briefly busy
+# is worse than no collector.
+scrape() {
+ curl -s --max-time 5 --unix-socket "${NODE_SOCKET}" http://localhost/metrics 2>/dev/null || true
+}
+
+agents_seen() {
+ scrape | awk '/^sam_node_agents_seen /{printf "%d", $2; found=1} END{if(!found) printf "0"}'
+}
+
+node_metric() {
+ scrape | awk -v m="$1" '$1 == m {printf "%d", $2; found=1} END{if(!found) printf "0"}'
+}
+
+echo "collecting every ${INTERVAL}s for ${DURATION}s into ${OUT}" >&2
+printf '%-9s %-8s %-8s %-10s %-10s\n' "elapsed" "agents" "microvms" "guest_gib" "box_gib" >&2
+
+started="$(date +%s)"
+: > "${OUT}"
+
+while :; do
+ now="$(date +%s)"
+ elapsed=$((now - started))
+ [[ "${elapsed}" -ge "${DURATION}" ]] && break
+
+ agents="$(agents_seen)"
+ vms="$(count_of 'firecracker --api-sock')"
+ boxes="$(count_of 'sam-box run --socket')"
+ vm_rss_kb="$(rss_total 'firecracker --api-sock')"
+ box_rss_kb="$(rss_total 'sam-box run --socket')"
+ node_rss_kb="$(rss_total 'sam-node run')"
+ mem_avail_kb="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)"
+ load="$(awk '{print $1}' /proc/loadavg)"
+
+ printf '{"elapsed_s":%d,"agents_seen":%s,"microvms":%s,"boundaries":%s,' \
+ "${elapsed}" "${agents:-0}" "${vms:-0}" "${boxes:-0}" >> "${OUT}"
+ printf '"guest_rss_kb":%s,"boundary_rss_kb":%s,"node_rss_kb":%s,' \
+ "${vm_rss_kb}" "${box_rss_kb}" "${node_rss_kb}" >> "${OUT}"
+ printf '"mem_available_kb":%s,"load1":%s,"requests_in_flight":%s}\n' \
+ "${mem_avail_kb}" "${load}" "$(node_metric sam_node_requests_in_flight)" >> "${OUT}"
+
+ printf '%-9s %-8s %-8s %-10s %-10s\n' \
+ "${elapsed}s" "${agents:-0}" "${vms:-0}" \
+ "$(awk -v k="${vm_rss_kb}" 'BEGIN{printf "%.2f", k/1048576}')" \
+ "$(awk -v k="${box_rss_kb}" 'BEGIN{printf "%.2f", k/1048576}')" >&2
+
+ sleep "${INTERVAL}"
+done
+
+echo >&2
+echo "=== summary ===" >&2
+awk -F'[:,]' '
+ /agents_seen/ {
+ for (i = 1; i <= NF; i++) {
+ if ($i ~ /"agents_seen"/) a = $(i+1)
+ if ($i ~ /"microvms"/) v = $(i+1)
+ if ($i ~ /"guest_rss_kb"/) g = $(i+1)
+ if ($i ~ /"boundary_rss_kb"/) b = $(i+1)
+ }
+ if (a > maxa) { maxa = a; maxg = g; maxb = b; maxv = v }
+ }
+ END {
+ printf "peak agents served: %d\n", maxa
+ printf "microVMs at peak: %d\n", maxv
+ if (maxa > 0) {
+ printf "guest memory: %.2f GiB (%.0f MiB per agent)\n", maxg/1048576, maxg/1024/maxa
+ printf "boundary memory: %.2f GiB (%.0f MiB per agent)\n", maxb/1048576, maxb/1024/maxa
+ printf "total per agent: %.0f MiB\n", (maxg+maxb)/1024/maxa
+ }
+ }
+' "${OUT}" >&2
+
+echo >&2
+echo "samples in ${OUT}" >&2
diff --git a/tests/scale/validate-launcher.sh b/tests/scale/validate-launcher.sh
index cec20210..16aa0c94 100755
--- a/tests/scale/validate-launcher.sh
+++ b/tests/scale/validate-launcher.sh
@@ -80,14 +80,14 @@ echo "==> Waiting for the agents to reach the mesh"
# The claim under test: one node, N boundaries, N distinct agents. Anything
# less means a sandbox never got out, and anything more means identities are
# leaking across sandboxes.
-seen=0
-for _ in $(seq 1 120); do
- seen="$(docker exec "${MESH_PREFIX}-node-1" \
- curl -s --unix-socket /sockets/node.sock http://localhost/metrics 2>/dev/null \
- | awk '/^sam_node_agents_seen /{printf "%d", $2}' || echo 0)"
- [[ "${seen:-0}" -ge "${COUNT}" ]] && break
- sleep 1
-done
+#
+# Collected the same way the fleet run collects it, so the reporting path is
+# exercised here rather than for the first time on a cloud machine.
+"${REPO_ROOT}/tests/scale/collect-fleet.sh" \
+ --node-socket "${MESH_SOCKET_DIR}/node.sock" \
+ --duration 120 --interval 5 --out "${WORK}/fleet.jsonl" || true
+
+seen="$(awk -F'[:,]' '/agents_seen/ {for (i=1;i<=NF;i++) if ($i ~ /"agents_seen"/ && $(i+1)+0 > m) m = $(i+1)+0} END {print m+0}' "${WORK}/fleet.jsonl")"
echo
echo "agents the node is serving: ${seen:-0} (expected ${COUNT})"