Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
nano-init: prefer a user namespace, and stop claiming a device plugin
Antonio Ojea6d5db06a8872
diff --git a/cmd/nano-init/isolation.go b/cmd/nano-init/isolation.goindex b323db6d..81da3c5a 100644--- a/cmd/nano-init/isolation.go+++ b/cmd/nano-init/isolation.go@@ -99,17 +99,21 @@ func tunHint(err error, statErr error) string {return "there is no " + tunDevice + ". In a microVM that means a guest kernel built without CONFIG_TUN" +" (the stock Firecracker CI kernels carry vsock but no tun driver; 6.18.41 has it)." +" In a container it means the device was not passed in: `--device /dev/net/tun` for docker," +- " or a device plugin for a Kubernetes pod, since the device cgroup denies it by default"+ " or a hostPath volume of type CharDevice for a Kubernetes pod"case os.IsPermission(statErr):- return tunDevice + " exists but cannot be opened. Under Kubernetes the device cgroup denies it unless a" +- " device plugin grants it; a user namespace does not help, because opening the device is checked" +- " against the host and not against the namespace"+ return tunDevice + " exists but cannot be opened. Check the device's own permissions, and any" ++ " device cgroup or seccomp policy the runtime applies; note that a user namespace does not" ++ " help here, because opening the device is checked against the host and not the namespace"- case errors.Is(err, syscall.EPERM), errors.Is(err, syscall.EACCES):+ case errors.Is(err, syscall.EPERM), errors.Is(err, syscall.EACCES),+ // netlink formats this one rather than wrapping the errno, so there is+ // nothing for errors.Is to match on.+ err != nil && strings.Contains(err.Error(), "TUNSETIFF"):return "creating a tun was refused. It needs CAP_NET_ADMIN in the user namespace that owns this network" +- " namespace: `--cap-add NET_ADMIN` for docker, securityContext.capabilities.add: [NET_ADMIN] for a pod," +- " or a user namespace of your own, which grants it over the namespaces it owns"+ " namespace: a user namespace of your own grants it over the namespaces it owns, which is what" ++ " --create-namespaces relies on; otherwise `--cap-add NET_ADMIN` for docker, or" ++ " securityContext.capabilities.add: [NET_ADMIN] for a pod"case errors.Is(err, syscall.ENODEV):return "the kernel has no tun driver. Rebuild the guest kernel with CONFIG_TUN=y"diff --git a/cmd/nano-init/isolation_test.go b/cmd/nano-init/isolation_test.goindex 7e956917..38257b75 100644--- a/cmd/nano-init/isolation_test.go+++ b/cmd/nano-init/isolation_test.go@@ -47,16 +47,16 @@ func TestTunHint(t *testing.T) {want: "--device /dev/net/tun",},{- name: "a pod missing the device is pointed at a device plugin",+ name: "a pod missing the device is pointed at a hostPath",err: syscall.ENOENT,statErr: notExist,- want: "device plugin",+ want: "hostPath",},{name: "a device that cannot be opened is not blamed on capabilities",statErr: denied,err: syscall.EPERM,- want: "device cgroup",+ want: "cannot be opened",},{name: "a refused create names the capability",diff --git a/cmd/nano-init/main.go b/cmd/nano-init/main.goindex 9a43258b..c0ae81a5 100644--- a/cmd/nano-init/main.go+++ b/cmd/nano-init/main.go@@ -134,7 +134,11 @@ func run(createNS bool, boundarySocket, cmdName string, cmdArgs []string) {log.Fatalf("locate this binary to re-execute it: %v", err)}args := append([]string{"run", "--create-namespaces", boundarySocket, cmdName}, cmdArgs...)- os.Exit(runAgent(ctx, cancel, self, args, withNamespaces(userNS)))+ code, err := runAgent(ctx, cancel, self, args, withNamespaces(userNS))+ if err != nil {+ log.Fatalf("create the sandbox namespaces: %v\n%s", err, namespaceHint(err))+ }+ os.Exit(code)}if createNS {@@ -163,7 +167,11 @@ func run(createNS bool, boundarySocket, cmdName string, cmdArgs []string) {log.Fatalf("set up sandbox network: %v", err)}- os.Exit(runAgent(ctx, cancel, cmdName, cmdArgs))+ code, err := runAgent(ctx, cancel, cmdName, cmdArgs)+ if err != nil {+ log.Fatalf("start agent: %v", err)+ }+ os.Exit(code)}// setupNetwork builds the only route out of the sandbox.@@ -244,7 +252,7 @@ func setupNetwork(ctx context.Context, boundarySocket string, names *resolver) e// The same supervision serves the namespace trampoline, whose child is this// binary again: orphans still reparent here and still have to be reaped, and// the exit code still has to be the one the caller sees.-func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cmdArgs []string, opts ...func(*exec.Cmd)) int {+func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cmdArgs []string, opts ...func(*exec.Cmd)) (int, error) {cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)cmd.Stdout, cmd.Stderr, cmd.Stdin = os.Stdout, os.Stderr, os.Stdincmd.Env = os.Environ() // Nothing injected: the agent is not configured, it is routed.@@ -261,7 +269,9 @@ func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cmcmd.WaitDelay = 5 * time.Secondif err := cmd.Start(); err != nil {- log.Fatalf("start agent: %v", err)+ // Returned rather than fatal: the namespace trampoline starts this same+ // binary, and a refusal there means something quite different.+ return 0, err}// As PID 1 this process inherits every orphan in the sandbox, so it has to@@ -276,18 +286,18 @@ func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cmif waitErr != nil && errors.Is(waitErr, syscall.ECHILD) {status := <-agentExitif status.Signaled() {- return 128 + int(status.Signal())+ return 128 + int(status.Signal()), nil}- return status.ExitStatus()+ return status.ExitStatus(), nil}if waitErr != nil {var exitErr *exec.ExitErrorif errors.As(waitErr, &exitErr) {- return exitErr.ExitCode()+ return exitErr.ExitCode(), nil}- return 1+ return 1, nil}- return 0+ return 0, nil}// reapChildren collects orphans and remembers the agent's own status.diff --git a/cmd/nano-init/namespaces.go b/cmd/nano-init/namespaces.goindex b493f9c1..9e1361f9 100644--- a/cmd/nano-init/namespaces.go+++ b/cmd/nano-init/namespaces.go@@ -15,6 +15,7 @@package mainimport (+ "errors""fmt""os""os/exec"@@ -85,24 +86,47 @@ func withNamespaces(userNS bool) func(*exec.Cmd) {// needUserNamespace decides how to get permission to create a network// namespace, or explains why neither way is open.//-// A user namespace is not the goal, it is the fallback: inside one this process-// is root and holds CAP_SYS_ADMIN over the namespaces it then creates, which is-// what makes the pod profile work without the pod being privileged. When-// CAP_SYS_ADMIN is already held there is no reason to add the extra namespace-// and the uid mapping that comes with it.+// A user namespace is preferred rather than merely tolerated. Inside one this+// process is root over the namespaces it then creates, which supplies+// CAP_NET_ADMIN for building the tun as well as CAP_SYS_ADMIN for making the+// namespace at all. Taking the capability route instead needs both to have been+// granted: a container given CAP_SYS_ADMIN but not CAP_NET_ADMIN creates the+// namespace and then cannot build the route out of it, which is a worse failure+// than not starting.+//+// So the capability route is the fallback, for hosts where user namespaces are+// turned off.func needUserNamespace() (bool, error) {- switch has, err := hasCapSysAdmin(); {- case err != nil:+ userNSErr := userNamespacesAvailable()+ if userNSErr == nil {+ return true, nil+ }+ has, err := hasCapSysAdmin()+ if err != nil {return false, err- case has:+ }+ if has {return false, nil}- if err := userNamespacesAvailable(); err != nil {- return false, fmt.Errorf(- "cannot create a network namespace: this process has no CAP_SYS_ADMIN, and %w. "+- "Grant CAP_SYS_ADMIN, or allow unprivileged user namespaces", err)+ return false, fmt.Errorf(+ "cannot create a network namespace: %w, and this process has no CAP_SYS_ADMIN. "++ "Allow unprivileged user namespaces, or grant CAP_SYS_ADMIN and CAP_NET_ADMIN", userNSErr)+}++// namespaceHint explains a refusal to create the namespaces.+//+// The kernel says EPERM and stops there, but in a container the cause is+// usually a sandboxing policy rather than a missing capability, and those are+// not visible from in here.+func namespaceHint(err error) string {+ if !errors.Is(err, syscall.EPERM) && !errors.Is(err, syscall.EACCES) {+ return ""}- return true, nil+ return "This is usually the runtime's own sandboxing rather than a missing capability. " ++ "Docker's default seccomp profile blocks creating a user namespace, and its default " ++ "AppArmor profile blocks the mount that follows; Kubernetes applies neither unless asked, " ++ "so a pod normally needs no securityContext for this at all. Where a profile is enforced, " ++ "it has to permit unshare(CLONE_NEWUSER|CLONE_NEWNS) and mount."}// hasCapSysAdmin reads the effective capability set of this process.diff --git a/site/content/docs/agent-architecture.md b/site/content/docs/agent-architecture.mdindex 27285aff..4d716c20 100644--- a/site/content/docs/agent-architecture.md+++ b/site/content/docs/agent-architecture.md@@ -349,7 +349,7 @@ supported answers. Each must satisfy the same five preconditions:| netns with no way out | the container runtime | the guest kernel, given no network device | **nothing does** || private `resolv.conf` | the container's mount namespace | the guest rootfs | **nothing does** || boundary socket | bind-mounted UDS | vsock, surfacing on the host as `<uds>_<port>` | shared `emptyDir` |-| TUN | `--device /dev/net/tun --cap-add NET_ADMIN` | `CONFIG_TUN` in the guest kernel | device plugin |+| TUN | `--device /dev/net/tun --cap-add NET_ADMIN` | `CONFIG_TUN` in the guest kernel | `hostPath` of type `CharDevice` || PID 1 | the container's entrypoint | `/sbin/init` | the container's entrypoint || isolation strength | namespaces, shared kernel | own kernel | namespaces, shared kernel |@@ -360,6 +360,17 @@ one network namespace**, so there is no per-container equivalent ofof them. A pod profile therefore has to create those two namespaces for itself,inside the container, before `nano-init` starts.+The pod column needs no capabilities, which is worth stating because the obvious+guesses are both wrong. The device cgroup does not deny `/dev/net/tun`, so no+device plugin is involved and a bind mount is enough; and granting+`CAP_SYS_ADMIN` alone is actively worse than granting nothing, because the+namespace then gets created without the `CAP_NET_ADMIN` the tun needs. Creating+a user namespace first supplies both over the namespaces it owns, so that is the+path taken whenever user namespaces are available. What a pod does need is a+seccomp and AppArmor policy that permits `unshare(CLONE_NEWUSER|CLONE_NEWNS)`+and `mount` — Kubernetes applies neither profile by default, and Docker's+defaults block both.+Because the failure is silent -- `tun0` alongside `eth0` is a working network,not an error -- `nano-init` refuses to start in a namespace that has anyinterface other than loopback and its own tun, whichever profile is in use.diff --git a/site/content/docs/user/running-agents.md b/site/content/docs/user/running-agents.mdindex b7aef8db..3f15697e 100644--- a/site/content/docs/user/running-agents.md+++ b/site/content/docs/user/running-agents.md@@ -285,6 +285,110 @@ before it can explain why. Of the published CI kernels, 6.18 has it and 5.10 and6.1 do not; the `.config` is published next to each image, so this is worthchecking before building a rootfs around one.+## In a Kubernetes pod++The two profiles above are handed a network namespace with nowhere to go: one by+`--network none`, one by having its own kernel. **A pod gives you neither.**+Every container in a pod shares a single network namespace, so there is no+per-container `--network none`, and the `/etc/resolv.conf` the kubelet writes is+shared by all of them. Left alone, an agent container would sit on the pod's+network with the pod's DNS, and `nano-init` would add a `tun0` beside `eth0`+that the agent could simply route around.++So for this profile `nano-init` builds the namespaces itself, with+`--create-namespaces`:++```yaml+apiVersion: v1+kind: Pod+metadata:+ name: agent+spec:+ volumes:+ - name: sam-uds # the only thing the three containers share+ emptyDir: {}+ - name: scratch # somewhere to put a private resolv.conf+ emptyDir: {}+ - name: tun+ hostPath:+ path: /dev/net/tun+ type: CharDevice+ containers:+ # The mesh member. It owns the identity; the agent never sees it.+ - name: sam-node+ image: ghcr.io/google/sam-node:latest+ args:+ - run+ - --jwt-path=/var/run/secrets/tokens/sam-token+ - --bind-addr= # no TCP listener, so there is no token to leak+ - --socket-path=/var/run/sam/node.sock+ volumeMounts:+ - { name: sam-uds, mountPath: /var/run/sam }++ # The boundary. Reaches the mesh only by dialling the node's socket.+ - name: sam-box+ image: ghcr.io/google/sam-box:latest+ args:+ - run+ - --socket=/var/run/sam/agent.sock+ - --sidecar-socket=/var/run/sam/node.sock+ - --egress-allow=api.github.com+ volumeMounts:+ - { name: sam-uds, mountPath: /var/run/sam }++ # The sandbox. No capabilities, no credential, and after nano-init+ # starts, no network.+ - name: agent+ image: your-agent-image+ command: ["/usr/local/bin/nano-init", "run", "--create-namespaces",+ "/var/run/sam/agent.sock", "python3", "/app/agent.py"]+ volumeMounts:+ - { name: sam-uds, mountPath: /var/run/sam }+ - { name: scratch, mountPath: /tmp }+ - { name: tun, mountPath: /dev/net/tun }+```++Note what is **not** there: no `securityContext`, no added capabilities, no+privileged flag, and no device plugin. Three things are needed instead, and each+is easy to get wrong in a way that does not name itself.++**The TUN device, bind-mounted.** That is all — the `hostPath` above. It is+worth saying plainly because the obvious worry turns out not to apply: the+device cgroup does *not* deny `/dev/net/tun`, so `open()` succeeds without a+device plugin and without `--device`. What is actually gated is the `TUNSETIFF`+that follows, and that is a capability question rather than a device one.++**No capabilities, deliberately.** Creating a network namespace normally needs+`CAP_SYS_ADMIN`, and creating a tun needs `CAP_NET_ADMIN`. Rather than asking+for either, `nano-init` creates a **user namespace** first, where it is root and+holds both over the namespaces it then makes. Granting `CAP_SYS_ADMIN` alone is+in fact worse than granting nothing: the namespace gets created, and then the+tun fails for want of `CAP_NET_ADMIN`.++**A seccomp and AppArmor policy that permits it.** This is the one that bites.+Creating a user namespace and remounting `/` as private are exactly the+operations container sandboxing profiles restrict. Kubernetes applies neither+profile unless you ask it to, so a default pod works — but a cluster with+`seccompDefault=RuntimeDefault`, or a `runtime/default` AppArmor profile, must+permit `unshare(CLONE_NEWUSER|CLONE_NEWNS)` and `mount`. Docker's defaults block+both, which is why the same image needs `--security-opt seccomp=unconfined+--security-opt apparmor=unconfined` to be tried locally.++**Somewhere writable.** A new mount namespace copies the mount table, not the+files behind it, so a private `resolv.conf` is a bind mount over a real file,+which has to be created somewhere. The `scratch` volume is that somewhere. With+a read-only root filesystem and no writable path, `nano-init` says so and stops.++What the agent sees once this is running is what it would see in a microVM:++```console+$ kubectl exec -c agent agent -- ip -o link show | cut -d: -f2+ lo+ tun0+```++No `eth0`, on a pod that has one.+## Why this shapeThe tempting alternative is to give the agent a token and a proxy and trust itdiff --git a/tests/integration/sandbox_namespaces_test.go b/tests/integration/sandbox_namespaces_test.gonew file mode 100644index 00000000..9b8279e5--- /dev/null+++ b/tests/integration/sandbox_namespaces_test.go@@ -0,0 +1,194 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package integration_test++import (+ "context"+ "fmt"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "net/url"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/google/sam/internal/sambox"+)++// TestSandboxCreatesItsOwnNamespaces covers the Kubernetes profile, which is+// the one no runtime isolates.+//+// The other two profiles are handed a network namespace with nowhere to go:+// `docker run --network none` makes one, and a microVM has its own kernel. A+// pod makes neither -- every container in a pod shares one network namespace,+// and the resolv.conf the kubelet writes is shared with them too -- so+// nano-init has to build both for itself.+//+// This runs it the way a pod would: from a namespace that has real interfaces+// and a real resolv.conf, with no external unshare and no privileges. What is+// being asserted is that the agent ends up somewhere else entirely, and that+// the boundary still reaches it there.+//+// It needs /dev/net/tun and unprivileged user namespaces, and skips without+// them rather than passing quietly.+func TestSandboxCreatesItsOwnNamespaces(t *testing.T) {+ const (+ allowedHost = "allowed.example"+ blockedHost = "blocked.example"+ serverBody = "reached the far side"+ )++ if _, err := os.Stat("/dev/net/tun"); err != nil {+ t.Skipf("this profile needs /dev/net/tun, the same as a pod does: %v", err)+ }++ nanoInitBin := buildBinary(t, "./cmd/nano-init")++ // The namespace this test runs in is emphatically not a sandbox, which is+ // the point: it stands in for the pod's shared namespace. If nano-init+ // used it, the assertions below would fail rather than quietly pass.+ if !hasForeignInterfaces(t) {+ t.Skip("this namespace has no interfaces, so it cannot stand in for a pod's")+ }++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = io.WriteString(w, serverBody)+ }))+ defer server.Close()++ serverURL, err := url.Parse(server.URL)+ if err != nil {+ t.Fatalf("parsing the server URL: %v", err)+ }++ sockDir, err := os.MkdirTemp("", "podsandbox")+ if err != nil {+ t.Fatalf("MkdirTemp: %v", err)+ }+ defer func() { _ = os.RemoveAll(sockDir) }()+ agentSocket := filepath.Join(sockDir, "agent.sock")++ var dialed struct {+ sync.Mutex+ names []string+ }++ egress, err := sambox.NewEgressPolicy([]string{allowedHost})+ if err != nil {+ t.Fatalf("NewEgressPolicy: %v", err)+ }+ listener, err := sambox.ListenSandboxSocket(agentSocket)+ if err != nil {+ t.Fatalf("ListenSandboxSocket: %v", err)+ }++ boundary := &sambox.SOCKS5Server{+ Dialer: &sambox.AgentDialer{+ Router: &sambox.Router{Egress: egress},+ DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {+ host, _, err := net.SplitHostPort(address)+ if err != nil {+ host = address+ }+ dialed.Lock()+ dialed.names = append(dialed.names, host)+ dialed.Unlock()+ return (&net.Dialer{}).DialContext(ctx, network, serverURL.Host)+ },+ },+ }++ ctx, cancel := context.WithCancel(context.Background())+ served := make(chan struct{})+ go func() {+ defer close(served)+ if err := boundary.Serve(ctx, listener); err != nil {+ t.Errorf("boundary: %v", err)+ }+ }()+ defer func() {+ cancel()+ <-served+ }()++ // The agent reports what it can see before using it, so a failure says+ // whether the namespace was wrong or the datapath was.+ agent := fmt.Sprintf(+ `echo "links=$(ip -o link show | cut -d: -f2 | tr -d " " | paste -sd,)"; `++ `echo "resolv=$(cat /etc/resolv.conf)"; `++ `curl -sS --max-time 20 http://%s/; echo "allowed-exit=$?"; `++ `curl -sS --max-time 20 http://%s/; echo "blocked-exit=$?"`,+ allowedHost, blockedHost,+ )++ runCtx, runCancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer runCancel()++ out, err := exec.CommandContext(runCtx, nanoInitBin,+ "run", "--create-namespaces", agentSocket, "bash", "-c", agent).CombinedOutput()+ t.Logf("nano-init exited with %v, sandbox output:\n%s", err, out)++ got := string(out)+ if strings.Contains(got, "user namespaces are disabled") ||+ strings.Contains(got, "unprivileged user namespaces are disabled") {+ t.Skipf("this host does not allow the namespaces a pod profile needs:\n%s", got)+ }++ // The whole claim. A pod's namespace has eth0 and whatever else the CNI+ // left; the agent must be somewhere with neither.+ if !strings.Contains(got, "links=lo,tun0") {+ t.Errorf("the agent did not get a namespace of its own; it reported:\n%s", got)+ }+ if !strings.Contains(got, "resolv=nameserver 169.254.1.1") {+ t.Errorf("the agent kept the outer resolv.conf, so a pod's DNS would have been repointed:\n%s", got)+ }++ // And the boundary still reaches it, which is what makes the isolation+ // useful rather than merely complete.+ if !strings.Contains(got, serverBody) || !strings.Contains(got, "allowed-exit=0") {+ t.Errorf("the sandbox did not reach %s through the boundary", allowedHost)+ }+ if strings.Contains(got, "blocked-exit=0") {+ t.Errorf("reaching %s succeeded, want the boundary to refuse it", blockedHost)+ }++ dialed.Lock()+ defer dialed.Unlock()+ if len(dialed.names) != 1 || dialed.names[0] != allowedHost {+ t.Errorf("the boundary dialled %q, want exactly [%q]", dialed.names, allowedHost)+ }+}++// hasForeignInterfaces reports whether this namespace has anything but+// loopback, which is what makes it a stand-in for a pod's.+func hasForeignInterfaces(t *testing.T) bool {+ t.Helper()+ links, err := net.Interfaces()+ if err != nil {+ t.Fatalf("list interfaces: %v", err)+ }+ for _, l := range links {+ if l.Name != "lo" {+ return true+ }+ }+ return false+}