Skip to content

Agent Mesh

tests: exercise the routed sandbox, in a container and a microVM

Antonio Ojea25b679050286
diff --git a/cmd/nano-init/go.mod b/cmd/nano-init/go.mod
index 406c8feb..97b42e52 100644
--- a/cmd/nano-init/go.mod
+++ b/cmd/nano-init/go.mod
@@ -1,6 +1,6 @@
module github.com/google/sam/cmd/nano-init
-go 1.26.5
+go 1.26.3
require (
github.com/mdlayher/vsock v1.3.0
diff --git a/tests/e2e/agent_sandbox.bats b/tests/e2e/agent_sandbox.bats
index c4a7c413..1d394701 100644
--- a/tests/e2e/agent_sandbox.bats
+++ b/tests/e2e/agent_sandbox.bats
@@ -18,11 +18,12 @@ teardown() {
mesh_cleanup_test_resources
}
-# agent_curl runs a command inside the agent sandbox, which reaches the boundary
-# over the socket and nothing else.
+# agent_curl runs a command inside the agent sandbox. It is an ordinary curl
+# with no proxy flag: the sandbox has one route and it leads to the boundary, so
+# reaching the mesh takes no cooperation from the client.
agent_curl() {
docker exec "${MESH_PREFIX}-agent" curl -s -o /dev/null -w '%{http_code}' \
- --max-time 10 --socks5-hostname 127.0.0.1:1080 "$@"
+ --max-time 10 "$@"
}
@test "an agent with no network reaches the mesh through its socket, and nothing else" {
@@ -62,16 +63,31 @@ agent_curl() {
false
}
- # The sandbox: no network, no credential, only the boundary socket. socat
- # stands in for the tun2socks a real sandbox runs, giving an ordinary client a
- # TCP address for a socket.
+ # The sandbox: no network, no credential, no proxy configuration. nano-init is
+ # PID 1 and builds the only route out, so the client above is unmodified and
+ # does not know it is confined. NET_ADMIN and /dev/net/tun are what building
+ # that route costs.
docker run -d --name "${MESH_PREFIX}-agent" \
--network none \
- --user "$(id -u):$(id -g)" \
+ --cap-add NET_ADMIN \
+ --device /dev/net/tun \
+ --user 0:0 \
-v "${MESH_SOCKET_DIR}:/sockets" \
+ --entrypoint /usr/local/bin/nano-init \
"${MESH_RUNTIME_IMAGE}" \
- socat TCP-LISTEN:1080,fork,reuseaddr UNIX-CONNECT:/sockets/agent.sock >/dev/null
- sleep 1
+ run /sockets/agent.sock sleep 300 >/dev/null
+
+ # Wait for the route rather than sleeping: a request issued before tun0 is up
+ # fails for a reason that has nothing to do with the mesh.
+ local waited=0
+ until docker exec "${MESH_PREFIX}-agent" ip route show default 2>/dev/null | grep -q tun0; do
+ sleep 0.2
+ waited=$((waited + 1))
+ [[ "${waited}" -lt 50 ]] || {
+ docker logs "${MESH_PREFIX}-agent"
+ false
+ }
+ done
# What an agent is for: the mesh's inference and tool endpoints, by name.
run agent_curl "http://mesh.sam.alt/v1/models"
@@ -100,9 +116,8 @@ agent_curl() {
}
done
- # Egress is deny-by-default, and a name nobody allowed fails the SOCKS5
- # handshake rather than being dialled.
- run docker exec "${MESH_PREFIX}-agent" curl -s --max-time 10 \
- --socks5-hostname 127.0.0.1:1080 "http://blocked.invalid/"
+ # Egress is deny-by-default, and a name nobody allowed is refused by the
+ # boundary rather than dialled.
+ run docker exec "${MESH_PREFIX}-agent" curl -s --max-time 10 "http://blocked.invalid/"
[[ "$status" -ne 0 ]]
}
diff --git a/tests/e2e/docker/Dockerfile.sam-runtime b/tests/e2e/docker/Dockerfile.sam-runtime
index 57a1f5fa..0f89a369 100644
--- a/tests/e2e/docker/Dockerfile.sam-runtime
+++ b/tests/e2e/docker/Dockerfile.sam-runtime
@@ -1,7 +1,9 @@
# syntax=docker/dockerfile:1.7
# Build container: compiles the Go binaries via `make build`.
-FROM golang:1.25-alpine AS build
+# 1.26 because nano-init's TCP stack requires it; the root module is happy on
+# anything newer than it asks for.
+FROM golang:1.26-alpine AS build
WORKDIR /src
RUN apk add --no-cache git make
@@ -19,9 +21,10 @@ RUN --mount=type=cache,target=/go/pkg/mod \
# Final container: minimal alpine runtime hosting the built binaries.
FROM alpine:3.20
-# socat and curl let a test container stand in for an agent sandbox: socat
-# bridges a loopback port to the gateway's socket, and curl is the agent.
-RUN apk add --no-cache busybox socat curl
+# curl is the agent in sandbox tests, and iproute2 lets a test assert what the
+# sandbox's routing table actually says. socat remains for tests that bridge a
+# socket by hand; nano-init no longer needs it.
+RUN apk add --no-cache busybox socat curl iproute2
RUN addgroup -S sam && adduser -S -G sam sam
diff --git a/tests/e2e/microvm_sandbox.bats b/tests/e2e/microvm_sandbox.bats
new file mode 100644
index 00000000..c5e0dea8
--- /dev/null
+++ b/tests/e2e/microvm_sandbox.bats
@@ -0,0 +1,115 @@
+#!/usr/bin/env bats
+
+# The microVM datapath, end to end, with a real kernel booting a real guest.
+#
+# Everything else about the sandbox is covered more cheaply elsewhere: the
+# boundary has unit tests, the routed container path has agent_sandbox.bats, and
+# the vsock dialer is exercised against loopback in the nano-init module. What
+# none of those touch is the one arrangement the scale run depends on -- a guest
+# with no network device reaching a Unix socket on the host through Firecracker's
+# vsock, where the host side is named "<uds_path>_<port>" and getting that name
+# wrong fails silently.
+#
+# This deliberately does not stand up a mesh. A mesh would make the test slower
+# and would not make it prove anything more: what is in question is whether a
+# microVM can reach the boundary at all.
+
+load "lib/container_mesh.bash"
+
+FC_DIR=""
+FC_PID=""
+
+setup() {
+ command -v firecracker >/dev/null 2>&1 || skip "firecracker is not installed"
+ [[ -r /dev/kvm && -w /dev/kvm ]] || skip "no writable /dev/kvm on this machine"
+ [[ -f "${FC_KERNEL:-/opt/microvm/vmlinux.bin}" ]] || skip "no guest kernel; set FC_KERNEL"
+ [[ -f "${FC_ROOTFS:-/opt/microvm/rootfs.ext4}" ]] || skip "no guest rootfs; set FC_ROOTFS (scripts/build-rootfs.sh)"
+
+ FC_DIR="$(mktemp -d /tmp/fc-smoke-XXXXXX)"
+}
+
+teardown() {
+ [[ -n "${FC_PID}" ]] && kill "${FC_PID}" 2>/dev/null
+ [[ -n "${FC_DIR}" ]] && rm -rf "${FC_DIR}"
+ return 0
+}
+
+fc_put() {
+ curl -sf -X PUT --unix-socket "${FC_DIR}/fc.sock" \
+ "http://localhost$1" -H 'Content-Type: application/json' -d "$2" >/dev/null
+}
+
+@test "a microVM with no network device reaches the boundary over vsock" {
+ local kernel="${FC_KERNEL:-/opt/microvm/vmlinux.bin}"
+ local rootfs="${FC_ROOTFS:-/opt/microvm/rootfs.ext4}"
+
+ # The boundary stands in for sam-box: it only has to accept, because what is
+ # being tested is whether a connection arrives at all. Firecracker multiplexes
+ # guest connections onto "<uds>_<port>", so this listens on the port-suffixed
+ # name and nothing else will do.
+ local vsock_uds="${FC_DIR}/vm.vsock"
+ socat -u "UNIX-LISTEN:${vsock_uds}_1080,fork" "CREATE:${FC_DIR}/arrived" &
+ local boundary_pid=$!
+
+ cp "${rootfs}" "${FC_DIR}/rootfs.ext4"
+
+ firecracker --api-sock "${FC_DIR}/fc.sock" >"${FC_DIR}/fc.log" 2>&1 &
+ FC_PID=$!
+
+ local waited=0
+ until [[ -S "${FC_DIR}/fc.sock" ]]; do
+ sleep 0.1
+ waited=$((waited + 1))
+ [[ "${waited}" -lt 50 ]] || {
+ cat "${FC_DIR}/fc.log"
+ false
+ }
+ done
+
+ fc_put /boot-source "{
+ \"kernel_image_path\": \"${kernel}\",
+ \"boot_args\": \"console=ttyS0 reboot=k panic=1 pci=off\"
+ }"
+ fc_put /machine-config '{ "vcpu_count": 1, "mem_size_mib": 256 }'
+ fc_put /drives/rootfs "{
+ \"drive_id\": \"rootfs\",
+ \"path_on_host\": \"${FC_DIR}/rootfs.ext4\",
+ \"is_root_device\": true,
+ \"is_read_only\": false
+ }"
+ fc_put /vsock "{
+ \"vsock_id\": \"vsock0\",
+ \"guest_cid\": 3,
+ \"uds_path\": \"${vsock_uds}\"
+ }"
+ fc_put /actions '{ "action_type": "InstanceStart" }'
+
+ # The guest boots, nano-init builds its tun and the agent resolves a mesh
+ # name, which can only leave through vsock. Something arriving here means the
+ # whole chain worked: no NIC in the guest, a route through tun0, gVisor
+ # terminating the connection, and the host side named correctly.
+ waited=0
+ until [[ -s "${FC_DIR}/arrived" ]]; do
+ sleep 0.5
+ waited=$((waited + 1))
+ [[ "${waited}" -lt 120 ]] || {
+ echo "nothing reached the boundary in 60s"
+ echo "--- firecracker ---"
+ cat "${FC_DIR}/fc.log"
+ kill "${boundary_pid}" 2>/dev/null
+ false
+ }
+ done
+
+ kill "${boundary_pid}" 2>/dev/null
+
+ # A SOCKS5 greeting, which is what nano-init opens with. Anything else means
+ # something arrived but it was not the boundary protocol.
+ run head -c 1 "${FC_DIR}/arrived"
+ [[ "$status" -eq 0 ]]
+ printf '%s' "$output" | od -An -tu1 | grep -qE '\<5\>' || {
+ echo "first byte was not a SOCKS5 version marker:"
+ od -An -tx1 -N16 "${FC_DIR}/arrived"
+ false
+ }
+}