Skip to content

Agent Mesh

scale: run the fleet launcher for real, and fix what that found

Antonio Ojead4958f6bf92f
diff --git a/development/examples/agent-harness/agent.py b/development/examples/agent-harness/agent.py
index 459adfea..ca593d7d 100644
--- a/development/examples/agent-harness/agent.py
+++ b/development/examples/agent-harness/agent.py
@@ -31,7 +31,7 @@ import os
import sys
from mcp import ClientSession
-from mcp.client.sse import sse_client
+from mcp.client.streamable_http import streamable_http_client
from openai import AsyncOpenAI
# The mesh's own name for itself. Not DNS, not routable, and deliberately not
@@ -93,7 +93,10 @@ async def run(task, model, max_steps):
# the mesh. The SDK requires the argument, so it gets a placeholder.
client = AsyncOpenAI(base_url=f"{MESH}/v1", api_key="unused")
- async with sse_client(f"{MESH}/mcp") as (read, write):
+ # Streamable HTTP, which is what the mesh serves. The older SSE transport
+ # is answered with 400, and that arrives late enough to look like a network
+ # problem rather than a protocol one.
+ async with streamable_http_client(f"{MESH}/mcp") as (read, write):
async with ClientSession(read, write) as session:
await session.initialize()
diff --git a/development/examples/agent-harness/requirements.txt b/development/examples/agent-harness/requirements.txt
index bbf20ca7..a0e09d86 100644
--- a/development/examples/agent-harness/requirements.txt
+++ b/development/examples/agent-harness/requirements.txt
@@ -5,5 +5,8 @@
# There is no SOCKS support here on purpose. The sandbox routes the agent's
# traffic to the boundary through tun0, so the agent neither configures a proxy
# nor knows there is one.
-openai>=1.40
-mcp>=1.2
+openai>=1.40,<3
+# Pinned to a major version: 1.x spells the transport streamablehttp_client and
+# 2.x spells it streamable_http_client, so an unpinned floor silently changes
+# the API underneath a sandbox image.
+mcp>=2.0,<3
diff --git a/scripts/launch-microvms.sh b/scripts/launch-microvms.sh
index a7981906..cbe5b829 100755
--- a/scripts/launch-microvms.sh
+++ b/scripts/launch-microvms.sh
@@ -39,34 +39,47 @@ if [ ! -d "$WORKDIR" ]; then
exit 1
fi
-NODE_UDS="/var/run/sam-node.sock"
-NODE_DIR="/var/lib/sam-node"
+# Everything the host writes is overridable, so this can be exercised on a
+# workstation before it is trusted on a fleet. The defaults are the paths a
+# provisioned VM has.
+NODE_UDS="${NODE_UDS:-/var/run/sam-node.sock}"
+NODE_DIR="${NODE_DIR:-/var/lib/sam-node}"
+RUN_DIR="${RUN_DIR:-/var/run}"
+LOG_DIR="${LOG_DIR:-/var/log}"
+SAM_NODE="${SAM_NODE:-sam-node}"
+SAM_BOX="${SAM_BOX:-sam-box}"
+
+mkdir -p "${RUN_DIR}" "${LOG_DIR}"
echo "=== One node for the host ==="
-mkdir -p "$NODE_DIR"
-if ! pgrep -f "sam-node run .*${NODE_UDS}" >/dev/null; then
- rm -f "$NODE_UDS"
- sam-node run \
- --data-dir "$NODE_DIR" \
+# An existing socket is taken as an existing node. That makes this safe to run
+# twice, and lets an operator start the node however their environment needs
+# rather than only the way this script would.
+if [ -S "${NODE_UDS}" ]; then
+ echo "Using the node already serving ${NODE_UDS}"
+else
+ mkdir -p "${NODE_DIR}"
+ "${SAM_NODE}" run \
+ --data-dir "${NODE_DIR}" \
--control-plane "$CONTROL_PLANE" \
--bind-addr "" \
- --socket-path "$NODE_UDS" \
- > /var/log/sam-node.log 2>&1 &
+ --socket-path "${NODE_UDS}" \
+ > "${LOG_DIR}/sam-node.log" 2>&1 &
# The boundaries are useless before the node answers, and starting a
# thousand of them against a socket that does not exist yet produces a
# thousand identical failures instead of one clear one.
for _ in $(seq 1 600); do
- [ -S "$NODE_UDS" ] && break
+ [ -S "${NODE_UDS}" ] && break
sleep 0.1
done
- [ -S "$NODE_UDS" ] || {
- echo "node never bound $NODE_UDS" >&2
- tail -20 /var/log/sam-node.log >&2
+ [ -S "${NODE_UDS}" ] || {
+ echo "node never bound ${NODE_UDS}" >&2
+ tail -20 "${LOG_DIR}/sam-node.log" >&2
exit 1
}
+ echo "Node ready at ${NODE_UDS}"
fi
-echo "Node ready at $NODE_UDS"
fc_put() {
curl -sf -X PUT --unix-socket "$1" \
@@ -79,10 +92,10 @@ for i in $(seq 1 "$COUNT"); do
# Firecracker's vsock multiplexes guest connections onto
# "<uds_path>_<port>", so the boundary must listen on that exact name for
# the guest's connections to CID 2 port 1080 to arrive.
- VSOCK_UDS="/var/run/sam-$VM_ID.vsock"
+ VSOCK_UDS="${RUN_DIR}/sam-$VM_ID.vsock"
BOUNDARY_UDS="${VSOCK_UDS}_1080"
API_SOCKET="/tmp/firecracker-$VM_ID.socket"
- BUNDLE="/var/run/sam-$VM_ID.bundle.yaml"
+ BUNDLE="${RUN_DIR}/sam-$VM_ID.bundle.yaml"
rm -f "$VSOCK_UDS" "$BOUNDARY_UDS" "$API_SOCKET"
@@ -104,14 +117,14 @@ EOF
# There is no credential issuer in this harness, and the flag says so
# rather than a default quietly meaning it.
- sam-box run \
+ "${SAM_BOX}" run \
--socket "$BOUNDARY_UDS" \
- --sidecar-socket "$NODE_UDS" \
+ --sidecar-socket "${NODE_UDS}" \
--bundle "$BUNDLE" \
--insecure-unverified-bundle \
- > "/var/log/sam-box-$VM_ID.log" 2>&1 &
+ > "${LOG_DIR}/sam-box-$VM_ID.log" 2>&1 &
- firecracker --api-sock "$API_SOCKET" > "/var/log/fc-$VM_ID.log" 2>&1 &
+ firecracker --api-sock "$API_SOCKET" > "${LOG_DIR}/fc-$VM_ID.log" 2>&1 &
for _ in $(seq 1 100); do
[ -S "$API_SOCKET" ] && break
diff --git a/tests/scale/validate-launcher.sh b/tests/scale/validate-launcher.sh
new file mode 100755
index 00000000..cec20210
--- /dev/null
+++ b/tests/scale/validate-launcher.sh
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Run the fleet launcher for real, small.
+#
+# launch-microvms.sh is what a provisioned VM runs, and until this existed it
+# had never been executed anywhere. A script that is only ever run for the first
+# time on a fleet of cloud machines is a script whose bugs are discovered at the
+# most expensive possible moment, so this runs it against a real mesh with a
+# couple of sandboxes and checks the thing that matters: that the node ends up
+# serving as many distinct agents as there are microVMs.
+#
+# Usage:
+# FC_KERNEL=... FC_ROOTFS=... FC_BIN=... tests/scale/validate-launcher.sh [count]
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "${REPO_ROOT}"
+
+COUNT="${1:-2}"
+WORK="$(mktemp -d /tmp/sam-launcher-XXXXXX)"
+
+export BATS_TEST_NAME="launcher"
+export BATS_TEST_NUMBER=0
+export BATS_TEST_DIRNAME="${REPO_ROOT}/tests/e2e"
+# shellcheck source=/dev/null
+source "${REPO_ROOT}/tests/e2e/lib/container_mesh.bash"
+
+cleanup() {
+ local status=$?
+ echo "==> Tearing down"
+ pkill -f "sam-box run --socket ${WORK}/" 2>/dev/null || true
+ pkill -f "firecracker --api-sock /tmp/firecracker-vm-" 2>/dev/null || true
+ mesh_cleanup_test_resources 2>/dev/null || true
+ rm -rf "${WORK}"
+ exit "${status}"
+}
+trap cleanup EXIT
+
+echo "==> Building binaries"
+make build >/dev/null
+
+echo "==> Standing up a mesh"
+mesh_setup_suite
+mesh_setup_env
+mesh_start_mock_oidc
+mesh_start_router
+mesh_start_node 1 "--socket-path /sockets/node.sock --data-dir /sockets/node-data" "" \
+ "-v ${MESH_SOCKET_DIR}:/sockets --user $(id -u):$(id -g) -e HOME=/sockets"
+mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 120
+mesh_wait_for_mcp_ready 1 30
+
+echo "==> Running the fleet launcher for ${COUNT} sandboxes"
+# The node is already up, and the launcher takes an existing socket as an
+# existing node, which is the same path a provisioned VM takes on a re-run.
+NODE_UDS="${MESH_SOCKET_DIR}/node.sock" \
+NODE_DIR="${WORK}/node-data" \
+RUN_DIR="${WORK}" \
+LOG_DIR="${WORK}" \
+SAM_BOX="${REPO_ROOT}/bin/sam-box" \
+WORKDIR="${FC_WORKDIR:-/tmp/microvm}" \
+AGENT_DOMAIN="validate.sam-mesh.dev" \
+ bash "${REPO_ROOT}/scripts/launch-microvms.sh" "${COUNT}"
+
+echo "==> Waiting for the agents to reach the mesh"
+# The claim under test: one node, N boundaries, N distinct agents. Anything
+# less means a sandbox never got out, and anything more means identities are
+# leaking across sandboxes.
+seen=0
+for _ in $(seq 1 120); do
+ seen="$(docker exec "${MESH_PREFIX}-node-1" \
+ curl -s --unix-socket /sockets/node.sock http://localhost/metrics 2>/dev/null \
+ | awk '/^sam_node_agents_seen /{printf "%d", $2}' || echo 0)"
+ [[ "${seen:-0}" -ge "${COUNT}" ]] && break
+ sleep 1
+done
+
+echo
+echo "agents the node is serving: ${seen:-0} (expected ${COUNT})"
+if [[ "${seen:-0}" -lt "${COUNT}" ]]; then
+ echo "--- boundary logs ---"
+ tail -n 5 "${WORK}"/sam-box-vm-*.log 2>/dev/null || true
+ echo "--- guest consoles ---"
+ grep -ahvE "^\[ *[0-9]+\.[0-9]+\]|^2026-" "${WORK}"/fc-vm-*.log 2>/dev/null | tail -20 || true
+ exit 1
+fi
+
+echo "OK: ${COUNT} sandboxes, one node, ${seen} distinct agents"