Skip to content

Agent Mesh

microvm: boot an agent sandbox, and fix what that found

Antonio Ojeada10ccc3b71a
diff --git a/cmd/nano-init/main.go b/cmd/nano-init/main.go
index 484db41a..f7cd037d 100644
--- a/cmd/nano-init/main.go
+++ b/cmd/nano-init/main.go
@@ -148,7 +148,11 @@ func setupNetwork(ctx context.Context, boundarySocket string, names *resolver) e
Mode: netlink.TUNTAP_MODE_TUN,
}
if err := netlink.LinkAdd(tun); err != nil {
- return fmt.Errorf("create %s: %w", tunName, err)
+ // The usual cause is a guest kernel built without CONFIG_TUN, which is
+ // the case for the stock Firecracker CI kernels: they carry vsock but
+ // no tun driver. Without it there is no way to give the sandbox a
+ // route, so this is fatal rather than something to work around.
+ return fmt.Errorf("create %s (does this kernel have CONFIG_TUN?): %w", tunName, err)
}
if err := netlink.LinkSetUp(tun); err != nil {
return fmt.Errorf("bring up %s: %w", tunName, err)
diff --git a/scripts/build-rootfs.sh b/scripts/build-rootfs.sh
index 900bc4bf..bfcfb7cc 100755
--- a/scripts/build-rootfs.sh
+++ b/scripts/build-rootfs.sh
@@ -1,6 +1,10 @@
#!/bin/bash
set -e
+# The Dockerfile below copies repo-relative paths, so the build context has to
+# be the repo root regardless of where this was invoked from.
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
# Usage: ./scripts/build-rootfs.sh [GCS_URL]
GCS_URL=${1:-}
diff --git a/scripts/microvm-init.sh b/scripts/microvm-init.sh
index c1d8b51e..860cce35 100755
--- a/scripts/microvm-init.sh
+++ b/scripts/microvm-init.sh
@@ -12,10 +12,26 @@
# port 1080 arrives on the host at the path sam-box serves.
set -eu
-mount -t proc proc /proc
-mount -t sysfs sysfs /sys
-mount -t devtmpfs devtmpfs /dev
+# Best effort, not preconditions: a kernel configured with devtmpfs has already
+# mounted /dev by the time init runs, and under `set -e` that redundant mount
+# is enough to kill PID 1 before it does anything.
+mount -t proc proc /proc 2>/dev/null || true
+mount -t sysfs sysfs /sys 2>/dev/null || true
+mount -t devtmpfs devtmpfs /dev 2>/dev/null || true
-exec /usr/local/bin/nano-init run vsock://2:1080 \
- python3 /app/agent/agent.py "${AGENT_TASK:-Describe the tools you have and what each is for.}" \
- > /dev/console 2>&1
+TASK="${AGENT_TASK:-Describe the tools you have and what each is for.}"
+
+# The kernel hands PID 1 an empty environment, so there is no PATH to inherit
+# and anything looked up by name is not found.
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# Not exec: when PID 1 exits the kernel panics immediately, and the panic
+# reaches the console before whatever init was trying to say about why it
+# failed. Holding the process open long enough for the console to drain is the
+# difference between a diagnosis and a stack trace.
+if ! /usr/local/bin/nano-init run vsock://2:1080 python3 /app/agent/agent.py "${TASK}" > /dev/console 2>&1; then
+ echo "sandbox init failed; see above" > /dev/console
+fi
+
+sleep 3
+reboot -f
diff --git a/scripts/startup-script.sh b/scripts/startup-script.sh
index 443f6c64..b99d9d61 100644
--- a/scripts/startup-script.sh
+++ b/scripts/startup-script.sh
@@ -56,9 +56,22 @@ else
curl --retry 5 --retry-connrefused --retry-delay 5 -sL https://sam-mesh.dev/install.sh | bash
fi
-# Download the uncompressed Linux kernel for Firecracker
+# Download the uncompressed Linux kernel for Firecracker.
+#
+# It has to be one built with CONFIG_TUN. An agent sandbox has no network
+# device and reaches the mesh through a tun, so a kernel without the driver
+# gives it no route at all and its init exits before it can say why. The
+# quickstart kernel this used to fetch does not have it, and neither do the
+# 5.10 or 6.1 CI kernels; 6.18 does. Each image has its .config published
+# beside it, so this is checkable rather than folklore.
mkdir -p /opt/microvm
-curl --retry 5 --retry-connrefused --retry-delay 5 -fsSL -o /opt/microvm/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin
+FC_KERNEL_URL="${FC_KERNEL_URL:-https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/20260819-0a745def42dd-0/x86_64/debug/vmlinux-6.18.41}"
+curl --retry 5 --retry-connrefused --retry-delay 5 -fsSL -o /opt/microvm/vmlinux.bin "${FC_KERNEL_URL}"
+
+if ! strings /opt/microvm/vmlinux.bin | grep -q "Universal TUN/TAP"; then
+ echo "FATAL: guest kernel has no TUN driver; agent sandboxes cannot build a route" >&2
+ exit 1
+fi
# Ensure launch script is executable (if it was downloaded via GCS)
chmod +x /opt/microvm/launch-microvms.sh || true
diff --git a/site/content/docs/user/running-agents.md b/site/content/docs/user/running-agents.md
index e018162c..22b74d92 100644
--- a/site/content/docs/user/running-agents.md
+++ b/site/content/docs/user/running-agents.md
@@ -274,6 +274,13 @@ connections onto **`<uds_path>_<port>`** on the host. A guest connecting to CID
why the same `sam-box` works for containers and microVMs with no code in it that
knows the difference.
+Another costs a day: **the guest kernel must have `CONFIG_TUN=y`**. The stock
+Firecracker CI kernels do not — they carry vsock and virtio and little else — so
+a sandbox on one of them cannot build a tun, has no route, and its init exits
+before it can explain why. Of the published CI kernels, 6.18 has it and 5.10 and
+6.1 do not; the `.config` is published next to each image, so this is worth
+checking before building a rootfs around one.
+
## Why this shape
The tempting alternative is to give the agent a token and a proxy and trust it
diff --git a/tests/e2e/microvm_sandbox.bats b/tests/e2e/microvm_sandbox.bats
index c5e0dea8..69913690 100644
--- a/tests/e2e/microvm_sandbox.bats
+++ b/tests/e2e/microvm_sandbox.bats
@@ -22,9 +22,18 @@ FC_PID=""
setup() {
command -v firecracker >/dev/null 2>&1 || skip "firecracker is not installed"
[[ -r /dev/kvm && -w /dev/kvm ]] || skip "no writable /dev/kvm on this machine"
- [[ -f "${FC_KERNEL:-/opt/microvm/vmlinux.bin}" ]] || skip "no guest kernel; set FC_KERNEL"
+
+ local kernel="${FC_KERNEL:-/opt/microvm/vmlinux.bin}"
+ [[ -f "${kernel}" ]] || skip "no guest kernel; set FC_KERNEL"
[[ -f "${FC_ROOTFS:-/opt/microvm/rootfs.ext4}" ]] || skip "no guest rootfs; set FC_ROOTFS (scripts/build-rootfs.sh)"
+ # An agent sandbox needs a tun to have any route at all, and the stock
+ # Firecracker CI kernels are built without CONFIG_TUN: they carry vsock and
+ # virtio and nothing else. Checked here because the alternative is a guest
+ # that panics with its explanation truncated by the panic itself.
+ strings "${kernel}" 2>/dev/null | grep -q "Universal TUN/TAP" \
+ || skip "guest kernel has no TUN driver; agent sandboxes need CONFIG_TUN"
+
FC_DIR="$(mktemp -d /tmp/fc-smoke-XXXXXX)"
}