Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
scale: run the fleet launcher for real, and fix what that found
Antonio Ojead4958f6bf92f
diff --git a/development/examples/agent-harness/agent.py b/development/examples/agent-harness/agent.pyindex 459adfea..ca593d7d 100644--- a/development/examples/agent-harness/agent.py+++ b/development/examples/agent-harness/agent.py@@ -31,7 +31,7 @@ import osimport sysfrom mcp import ClientSession-from mcp.client.sse import sse_client+from mcp.client.streamable_http import streamable_http_clientfrom openai import AsyncOpenAI# The mesh's own name for itself. Not DNS, not routable, and deliberately not@@ -93,7 +93,10 @@ async def run(task, model, max_steps):# the mesh. The SDK requires the argument, so it gets a placeholder.client = AsyncOpenAI(base_url=f"{MESH}/v1", api_key="unused")- async with sse_client(f"{MESH}/mcp") as (read, write):+ # Streamable HTTP, which is what the mesh serves. The older SSE transport+ # is answered with 400, and that arrives late enough to look like a network+ # problem rather than a protocol one.+ async with streamable_http_client(f"{MESH}/mcp") as (read, write):async with ClientSession(read, write) as session:await session.initialize()diff --git a/development/examples/agent-harness/requirements.txt b/development/examples/agent-harness/requirements.txtindex bbf20ca7..a0e09d86 100644--- a/development/examples/agent-harness/requirements.txt+++ b/development/examples/agent-harness/requirements.txt@@ -5,5 +5,8 @@# There is no SOCKS support here on purpose. The sandbox routes the agent's# traffic to the boundary through tun0, so the agent neither configures a proxy# nor knows there is one.-openai>=1.40-mcp>=1.2+openai>=1.40,<3+# Pinned to a major version: 1.x spells the transport streamablehttp_client and+# 2.x spells it streamable_http_client, so an unpinned floor silently changes+# the API underneath a sandbox image.+mcp>=2.0,<3diff --git a/scripts/launch-microvms.sh b/scripts/launch-microvms.shindex a7981906..cbe5b829 100755--- a/scripts/launch-microvms.sh+++ b/scripts/launch-microvms.sh@@ -39,34 +39,47 @@ if [ ! -d "$WORKDIR" ]; thenexit 1fi-NODE_UDS="/var/run/sam-node.sock"-NODE_DIR="/var/lib/sam-node"+# Everything the host writes is overridable, so this can be exercised on a+# workstation before it is trusted on a fleet. The defaults are the paths a+# provisioned VM has.+NODE_UDS="${NODE_UDS:-/var/run/sam-node.sock}"+NODE_DIR="${NODE_DIR:-/var/lib/sam-node}"+RUN_DIR="${RUN_DIR:-/var/run}"+LOG_DIR="${LOG_DIR:-/var/log}"+SAM_NODE="${SAM_NODE:-sam-node}"+SAM_BOX="${SAM_BOX:-sam-box}"++mkdir -p "${RUN_DIR}" "${LOG_DIR}"echo "=== One node for the host ==="-mkdir -p "$NODE_DIR"-if ! pgrep -f "sam-node run .*${NODE_UDS}" >/dev/null; then- rm -f "$NODE_UDS"- sam-node run \- --data-dir "$NODE_DIR" \+# An existing socket is taken as an existing node. That makes this safe to run+# twice, and lets an operator start the node however their environment needs+# rather than only the way this script would.+if [ -S "${NODE_UDS}" ]; then+ echo "Using the node already serving ${NODE_UDS}"+else+ mkdir -p "${NODE_DIR}"+ "${SAM_NODE}" run \+ --data-dir "${NODE_DIR}" \--control-plane "$CONTROL_PLANE" \--bind-addr "" \- --socket-path "$NODE_UDS" \- > /var/log/sam-node.log 2>&1 &+ --socket-path "${NODE_UDS}" \+ > "${LOG_DIR}/sam-node.log" 2>&1 &# The boundaries are useless before the node answers, and starting a# thousand of them against a socket that does not exist yet produces a# thousand identical failures instead of one clear one.for _ in $(seq 1 600); do- [ -S "$NODE_UDS" ] && break+ [ -S "${NODE_UDS}" ] && breaksleep 0.1done- [ -S "$NODE_UDS" ] || {- echo "node never bound $NODE_UDS" >&2- tail -20 /var/log/sam-node.log >&2+ [ -S "${NODE_UDS}" ] || {+ echo "node never bound ${NODE_UDS}" >&2+ tail -20 "${LOG_DIR}/sam-node.log" >&2exit 1}+ echo "Node ready at ${NODE_UDS}"fi-echo "Node ready at $NODE_UDS"fc_put() {curl -sf -X PUT --unix-socket "$1" \@@ -79,10 +92,10 @@ for i in $(seq 1 "$COUNT"); do# Firecracker's vsock multiplexes guest connections onto# "<uds_path>_<port>", so the boundary must listen on that exact name for# the guest's connections to CID 2 port 1080 to arrive.- VSOCK_UDS="/var/run/sam-$VM_ID.vsock"+ VSOCK_UDS="${RUN_DIR}/sam-$VM_ID.vsock"BOUNDARY_UDS="${VSOCK_UDS}_1080"API_SOCKET="/tmp/firecracker-$VM_ID.socket"- BUNDLE="/var/run/sam-$VM_ID.bundle.yaml"+ BUNDLE="${RUN_DIR}/sam-$VM_ID.bundle.yaml"rm -f "$VSOCK_UDS" "$BOUNDARY_UDS" "$API_SOCKET"@@ -104,14 +117,14 @@ EOF# There is no credential issuer in this harness, and the flag says so# rather than a default quietly meaning it.- sam-box run \+ "${SAM_BOX}" run \--socket "$BOUNDARY_UDS" \- --sidecar-socket "$NODE_UDS" \+ --sidecar-socket "${NODE_UDS}" \--bundle "$BUNDLE" \--insecure-unverified-bundle \- > "/var/log/sam-box-$VM_ID.log" 2>&1 &+ > "${LOG_DIR}/sam-box-$VM_ID.log" 2>&1 &- firecracker --api-sock "$API_SOCKET" > "/var/log/fc-$VM_ID.log" 2>&1 &+ firecracker --api-sock "$API_SOCKET" > "${LOG_DIR}/fc-$VM_ID.log" 2>&1 &for _ in $(seq 1 100); do[ -S "$API_SOCKET" ] && breakdiff --git a/tests/scale/validate-launcher.sh b/tests/scale/validate-launcher.shnew file mode 100755index 00000000..cec20210--- /dev/null+++ b/tests/scale/validate-launcher.sh@@ -0,0 +1,102 @@+#!/usr/bin/env bash+#+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# Run the fleet launcher for real, small.+#+# launch-microvms.sh is what a provisioned VM runs, and until this existed it+# had never been executed anywhere. A script that is only ever run for the first+# time on a fleet of cloud machines is a script whose bugs are discovered at the+# most expensive possible moment, so this runs it against a real mesh with a+# couple of sandboxes and checks the thing that matters: that the node ends up+# serving as many distinct agents as there are microVMs.+#+# Usage:+# FC_KERNEL=... FC_ROOTFS=... FC_BIN=... tests/scale/validate-launcher.sh [count]++set -euo pipefail++REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+cd "${REPO_ROOT}"++COUNT="${1:-2}"+WORK="$(mktemp -d /tmp/sam-launcher-XXXXXX)"++export BATS_TEST_NAME="launcher"+export BATS_TEST_NUMBER=0+export BATS_TEST_DIRNAME="${REPO_ROOT}/tests/e2e"+# shellcheck source=/dev/null+source "${REPO_ROOT}/tests/e2e/lib/container_mesh.bash"++cleanup() {+ local status=$?+ echo "==> Tearing down"+ pkill -f "sam-box run --socket ${WORK}/" 2>/dev/null || true+ pkill -f "firecracker --api-sock /tmp/firecracker-vm-" 2>/dev/null || true+ mesh_cleanup_test_resources 2>/dev/null || true+ rm -rf "${WORK}"+ exit "${status}"+}+trap cleanup EXIT++echo "==> Building binaries"+make build >/dev/null++echo "==> Standing up a mesh"+mesh_setup_suite+mesh_setup_env+mesh_start_mock_oidc+mesh_start_router+mesh_start_node 1 "--socket-path /sockets/node.sock --data-dir /sockets/node-data" "" \+ "-v ${MESH_SOCKET_DIR}:/sockets --user $(id -u):$(id -g) -e HOME=/sockets"+mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 120+mesh_wait_for_mcp_ready 1 30++echo "==> Running the fleet launcher for ${COUNT} sandboxes"+# The node is already up, and the launcher takes an existing socket as an+# existing node, which is the same path a provisioned VM takes on a re-run.+NODE_UDS="${MESH_SOCKET_DIR}/node.sock" \+NODE_DIR="${WORK}/node-data" \+RUN_DIR="${WORK}" \+LOG_DIR="${WORK}" \+SAM_BOX="${REPO_ROOT}/bin/sam-box" \+WORKDIR="${FC_WORKDIR:-/tmp/microvm}" \+AGENT_DOMAIN="validate.sam-mesh.dev" \+ bash "${REPO_ROOT}/scripts/launch-microvms.sh" "${COUNT}"++echo "==> Waiting for the agents to reach the mesh"+# The claim under test: one node, N boundaries, N distinct agents. Anything+# less means a sandbox never got out, and anything more means identities are+# leaking across sandboxes.+seen=0+for _ in $(seq 1 120); do+ seen="$(docker exec "${MESH_PREFIX}-node-1" \+ curl -s --unix-socket /sockets/node.sock http://localhost/metrics 2>/dev/null \+ | awk '/^sam_node_agents_seen /{printf "%d", $2}' || echo 0)"+ [[ "${seen:-0}" -ge "${COUNT}" ]] && break+ sleep 1+done++echo+echo "agents the node is serving: ${seen:-0} (expected ${COUNT})"+if [[ "${seen:-0}" -lt "${COUNT}" ]]; then+ echo "--- boundary logs ---"+ tail -n 5 "${WORK}"/sam-box-vm-*.log 2>/dev/null || true+ echo "--- guest consoles ---"+ grep -ahvE "^\[ *[0-9]+\.[0-9]+\]|^2026-" "${WORK}"/fc-vm-*.log 2>/dev/null | tail -20 || true+ exit 1+fi++echo "OK: ${COUNT} sandboxes, one node, ${seen} distinct agents"