Skip to content

Agent Mesh

nano-init: prefer a user namespace, and stop claiming a device plugin

Antonio Ojea6d5db06a8872
diff --git a/cmd/nano-init/isolation.go b/cmd/nano-init/isolation.go
index b323db6d..81da3c5a 100644
--- a/cmd/nano-init/isolation.go
+++ b/cmd/nano-init/isolation.go
@@ -99,17 +99,21 @@ func tunHint(err error, statErr error) string {
return "there is no " + tunDevice + ". In a microVM that means a guest kernel built without CONFIG_TUN" +
" (the stock Firecracker CI kernels carry vsock but no tun driver; 6.18.41 has it)." +
" In a container it means the device was not passed in: `--device /dev/net/tun` for docker," +
- " or a device plugin for a Kubernetes pod, since the device cgroup denies it by default"
+ " or a hostPath volume of type CharDevice for a Kubernetes pod"
case os.IsPermission(statErr):
- return tunDevice + " exists but cannot be opened. Under Kubernetes the device cgroup denies it unless a" +
- " device plugin grants it; a user namespace does not help, because opening the device is checked" +
- " against the host and not against the namespace"
+ return tunDevice + " exists but cannot be opened. Check the device's own permissions, and any" +
+ " device cgroup or seccomp policy the runtime applies; note that a user namespace does not" +
+ " help here, because opening the device is checked against the host and not the namespace"
- case errors.Is(err, syscall.EPERM), errors.Is(err, syscall.EACCES):
+ case errors.Is(err, syscall.EPERM), errors.Is(err, syscall.EACCES),
+ // netlink formats this one rather than wrapping the errno, so there is
+ // nothing for errors.Is to match on.
+ err != nil && strings.Contains(err.Error(), "TUNSETIFF"):
return "creating a tun was refused. It needs CAP_NET_ADMIN in the user namespace that owns this network" +
- " namespace: `--cap-add NET_ADMIN` for docker, securityContext.capabilities.add: [NET_ADMIN] for a pod," +
- " or a user namespace of your own, which grants it over the namespaces it owns"
+ " namespace: a user namespace of your own grants it over the namespaces it owns, which is what" +
+ " --create-namespaces relies on; otherwise `--cap-add NET_ADMIN` for docker, or" +
+ " securityContext.capabilities.add: [NET_ADMIN] for a pod"
case errors.Is(err, syscall.ENODEV):
return "the kernel has no tun driver. Rebuild the guest kernel with CONFIG_TUN=y"
diff --git a/cmd/nano-init/isolation_test.go b/cmd/nano-init/isolation_test.go
index 7e956917..38257b75 100644
--- a/cmd/nano-init/isolation_test.go
+++ b/cmd/nano-init/isolation_test.go
@@ -47,16 +47,16 @@ func TestTunHint(t *testing.T) {
want: "--device /dev/net/tun",
},
{
- name: "a pod missing the device is pointed at a device plugin",
+ name: "a pod missing the device is pointed at a hostPath",
err: syscall.ENOENT,
statErr: notExist,
- want: "device plugin",
+ want: "hostPath",
},
{
name: "a device that cannot be opened is not blamed on capabilities",
statErr: denied,
err: syscall.EPERM,
- want: "device cgroup",
+ want: "cannot be opened",
},
{
name: "a refused create names the capability",
diff --git a/cmd/nano-init/main.go b/cmd/nano-init/main.go
index 9a43258b..c0ae81a5 100644
--- a/cmd/nano-init/main.go
+++ b/cmd/nano-init/main.go
@@ -134,7 +134,11 @@ func run(createNS bool, boundarySocket, cmdName string, cmdArgs []string) {
log.Fatalf("locate this binary to re-execute it: %v", err)
}
args := append([]string{"run", "--create-namespaces", boundarySocket, cmdName}, cmdArgs...)
- os.Exit(runAgent(ctx, cancel, self, args, withNamespaces(userNS)))
+ code, err := runAgent(ctx, cancel, self, args, withNamespaces(userNS))
+ if err != nil {
+ log.Fatalf("create the sandbox namespaces: %v\n%s", err, namespaceHint(err))
+ }
+ os.Exit(code)
}
if createNS {
@@ -163,7 +167,11 @@ func run(createNS bool, boundarySocket, cmdName string, cmdArgs []string) {
log.Fatalf("set up sandbox network: %v", err)
}
- os.Exit(runAgent(ctx, cancel, cmdName, cmdArgs))
+ code, err := runAgent(ctx, cancel, cmdName, cmdArgs)
+ if err != nil {
+ log.Fatalf("start agent: %v", err)
+ }
+ os.Exit(code)
}
// setupNetwork builds the only route out of the sandbox.
@@ -244,7 +252,7 @@ func setupNetwork(ctx context.Context, boundarySocket string, names *resolver) e
// The same supervision serves the namespace trampoline, whose child is this
// binary again: orphans still reparent here and still have to be reaped, and
// the exit code still has to be the one the caller sees.
-func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cmdArgs []string, opts ...func(*exec.Cmd)) int {
+func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cmdArgs []string, opts ...func(*exec.Cmd)) (int, error) {
cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
cmd.Stdout, cmd.Stderr, cmd.Stdin = os.Stdout, os.Stderr, os.Stdin
cmd.Env = os.Environ() // Nothing injected: the agent is not configured, it is routed.
@@ -261,7 +269,9 @@ func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cm
cmd.WaitDelay = 5 * time.Second
if err := cmd.Start(); err != nil {
- log.Fatalf("start agent: %v", err)
+ // Returned rather than fatal: the namespace trampoline starts this same
+ // binary, and a refusal there means something quite different.
+ return 0, err
}
// As PID 1 this process inherits every orphan in the sandbox, so it has to
@@ -276,18 +286,18 @@ func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cm
if waitErr != nil && errors.Is(waitErr, syscall.ECHILD) {
status := <-agentExit
if status.Signaled() {
- return 128 + int(status.Signal())
+ return 128 + int(status.Signal()), nil
}
- return status.ExitStatus()
+ return status.ExitStatus(), nil
}
if waitErr != nil {
var exitErr *exec.ExitError
if errors.As(waitErr, &exitErr) {
- return exitErr.ExitCode()
+ return exitErr.ExitCode(), nil
}
- return 1
+ return 1, nil
}
- return 0
+ return 0, nil
}
// reapChildren collects orphans and remembers the agent's own status.
diff --git a/cmd/nano-init/namespaces.go b/cmd/nano-init/namespaces.go
index b493f9c1..9e1361f9 100644
--- a/cmd/nano-init/namespaces.go
+++ b/cmd/nano-init/namespaces.go
@@ -15,6 +15,7 @@
package main
import (
+ "errors"
"fmt"
"os"
"os/exec"
@@ -85,24 +86,47 @@ func withNamespaces(userNS bool) func(*exec.Cmd) {
// needUserNamespace decides how to get permission to create a network
// namespace, or explains why neither way is open.
//
-// A user namespace is not the goal, it is the fallback: inside one this process
-// is root and holds CAP_SYS_ADMIN over the namespaces it then creates, which is
-// what makes the pod profile work without the pod being privileged. When
-// CAP_SYS_ADMIN is already held there is no reason to add the extra namespace
-// and the uid mapping that comes with it.
+// A user namespace is preferred rather than merely tolerated. Inside one this
+// process is root over the namespaces it then creates, which supplies
+// CAP_NET_ADMIN for building the tun as well as CAP_SYS_ADMIN for making the
+// namespace at all. Taking the capability route instead needs both to have been
+// granted: a container given CAP_SYS_ADMIN but not CAP_NET_ADMIN creates the
+// namespace and then cannot build the route out of it, which is a worse failure
+// than not starting.
+//
+// So the capability route is the fallback, for hosts where user namespaces are
+// turned off.
func needUserNamespace() (bool, error) {
- switch has, err := hasCapSysAdmin(); {
- case err != nil:
+ userNSErr := userNamespacesAvailable()
+ if userNSErr == nil {
+ return true, nil
+ }
+ has, err := hasCapSysAdmin()
+ if err != nil {
return false, err
- case has:
+ }
+ if has {
return false, nil
}
- if err := userNamespacesAvailable(); err != nil {
- return false, fmt.Errorf(
- "cannot create a network namespace: this process has no CAP_SYS_ADMIN, and %w. "+
- "Grant CAP_SYS_ADMIN, or allow unprivileged user namespaces", err)
+ return false, fmt.Errorf(
+ "cannot create a network namespace: %w, and this process has no CAP_SYS_ADMIN. "+
+ "Allow unprivileged user namespaces, or grant CAP_SYS_ADMIN and CAP_NET_ADMIN", userNSErr)
+}
+
+// namespaceHint explains a refusal to create the namespaces.
+//
+// The kernel says EPERM and stops there, but in a container the cause is
+// usually a sandboxing policy rather than a missing capability, and those are
+// not visible from in here.
+func namespaceHint(err error) string {
+ if !errors.Is(err, syscall.EPERM) && !errors.Is(err, syscall.EACCES) {
+ return ""
}
- return true, nil
+ return "This is usually the runtime's own sandboxing rather than a missing capability. " +
+ "Docker's default seccomp profile blocks creating a user namespace, and its default " +
+ "AppArmor profile blocks the mount that follows; Kubernetes applies neither unless asked, " +
+ "so a pod normally needs no securityContext for this at all. Where a profile is enforced, " +
+ "it has to permit unshare(CLONE_NEWUSER|CLONE_NEWNS) and mount."
}
// hasCapSysAdmin reads the effective capability set of this process.
diff --git a/site/content/docs/agent-architecture.md b/site/content/docs/agent-architecture.md
index 27285aff..4d716c20 100644
--- a/site/content/docs/agent-architecture.md
+++ b/site/content/docs/agent-architecture.md
@@ -349,7 +349,7 @@ supported answers. Each must satisfy the same five preconditions:
| netns with no way out | the container runtime | the guest kernel, given no network device | **nothing does** |
| private `resolv.conf` | the container's mount namespace | the guest rootfs | **nothing does** |
| boundary socket | bind-mounted UDS | vsock, surfacing on the host as `<uds>_<port>` | shared `emptyDir` |
-| TUN | `--device /dev/net/tun --cap-add NET_ADMIN` | `CONFIG_TUN` in the guest kernel | device plugin |
+| TUN | `--device /dev/net/tun --cap-add NET_ADMIN` | `CONFIG_TUN` in the guest kernel | `hostPath` of type `CharDevice` |
| PID 1 | the container's entrypoint | `/sbin/init` | the container's entrypoint |
| isolation strength | namespaces, shared kernel | own kernel | namespaces, shared kernel |
@@ -360,6 +360,17 @@ one network namespace**, so there is no per-container equivalent of
of them. A pod profile therefore has to create those two namespaces for itself,
inside the container, before `nano-init` starts.
+The pod column needs no capabilities, which is worth stating because the obvious
+guesses are both wrong. The device cgroup does not deny `/dev/net/tun`, so no
+device plugin is involved and a bind mount is enough; and granting
+`CAP_SYS_ADMIN` alone is actively worse than granting nothing, because the
+namespace then gets created without the `CAP_NET_ADMIN` the tun needs. Creating
+a user namespace first supplies both over the namespaces it owns, so that is the
+path taken whenever user namespaces are available. What a pod does need is a
+seccomp and AppArmor policy that permits `unshare(CLONE_NEWUSER|CLONE_NEWNS)`
+and `mount` — Kubernetes applies neither profile by default, and Docker's
+defaults block both.
+
Because the failure is silent -- `tun0` alongside `eth0` is a working network,
not an error -- `nano-init` refuses to start in a namespace that has any
interface other than loopback and its own tun, whichever profile is in use.
diff --git a/site/content/docs/user/running-agents.md b/site/content/docs/user/running-agents.md
index b7aef8db..3f15697e 100644
--- a/site/content/docs/user/running-agents.md
+++ b/site/content/docs/user/running-agents.md
@@ -285,6 +285,110 @@ before it can explain why. Of the published CI kernels, 6.18 has it and 5.10 and
6.1 do not; the `.config` is published next to each image, so this is worth
checking before building a rootfs around one.
+## In a Kubernetes pod
+
+The two profiles above are handed a network namespace with nowhere to go: one by
+`--network none`, one by having its own kernel. **A pod gives you neither.**
+Every container in a pod shares a single network namespace, so there is no
+per-container `--network none`, and the `/etc/resolv.conf` the kubelet writes is
+shared by all of them. Left alone, an agent container would sit on the pod's
+network with the pod's DNS, and `nano-init` would add a `tun0` beside `eth0`
+that the agent could simply route around.
+
+So for this profile `nano-init` builds the namespaces itself, with
+`--create-namespaces`:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: agent
+spec:
+ volumes:
+ - name: sam-uds # the only thing the three containers share
+ emptyDir: {}
+ - name: scratch # somewhere to put a private resolv.conf
+ emptyDir: {}
+ - name: tun
+ hostPath:
+ path: /dev/net/tun
+ type: CharDevice
+ containers:
+ # The mesh member. It owns the identity; the agent never sees it.
+ - name: sam-node
+ image: ghcr.io/google/sam-node:latest
+ args:
+ - run
+ - --jwt-path=/var/run/secrets/tokens/sam-token
+ - --bind-addr= # no TCP listener, so there is no token to leak
+ - --socket-path=/var/run/sam/node.sock
+ volumeMounts:
+ - { name: sam-uds, mountPath: /var/run/sam }
+
+ # The boundary. Reaches the mesh only by dialling the node's socket.
+ - name: sam-box
+ image: ghcr.io/google/sam-box:latest
+ args:
+ - run
+ - --socket=/var/run/sam/agent.sock
+ - --sidecar-socket=/var/run/sam/node.sock
+ - --egress-allow=api.github.com
+ volumeMounts:
+ - { name: sam-uds, mountPath: /var/run/sam }
+
+ # The sandbox. No capabilities, no credential, and after nano-init
+ # starts, no network.
+ - name: agent
+ image: your-agent-image
+ command: ["/usr/local/bin/nano-init", "run", "--create-namespaces",
+ "/var/run/sam/agent.sock", "python3", "/app/agent.py"]
+ volumeMounts:
+ - { name: sam-uds, mountPath: /var/run/sam }
+ - { name: scratch, mountPath: /tmp }
+ - { name: tun, mountPath: /dev/net/tun }
+```
+
+Note what is **not** there: no `securityContext`, no added capabilities, no
+privileged flag, and no device plugin. Three things are needed instead, and each
+is easy to get wrong in a way that does not name itself.
+
+**The TUN device, bind-mounted.** That is all — the `hostPath` above. It is
+worth saying plainly because the obvious worry turns out not to apply: the
+device cgroup does *not* deny `/dev/net/tun`, so `open()` succeeds without a
+device plugin and without `--device`. What is actually gated is the `TUNSETIFF`
+that follows, and that is a capability question rather than a device one.
+
+**No capabilities, deliberately.** Creating a network namespace normally needs
+`CAP_SYS_ADMIN`, and creating a tun needs `CAP_NET_ADMIN`. Rather than asking
+for either, `nano-init` creates a **user namespace** first, where it is root and
+holds both over the namespaces it then makes. Granting `CAP_SYS_ADMIN` alone is
+in fact worse than granting nothing: the namespace gets created, and then the
+tun fails for want of `CAP_NET_ADMIN`.
+
+**A seccomp and AppArmor policy that permits it.** This is the one that bites.
+Creating a user namespace and remounting `/` as private are exactly the
+operations container sandboxing profiles restrict. Kubernetes applies neither
+profile unless you ask it to, so a default pod works — but a cluster with
+`seccompDefault=RuntimeDefault`, or a `runtime/default` AppArmor profile, must
+permit `unshare(CLONE_NEWUSER|CLONE_NEWNS)` and `mount`. Docker's defaults block
+both, which is why the same image needs `--security-opt seccomp=unconfined
+--security-opt apparmor=unconfined` to be tried locally.
+
+**Somewhere writable.** A new mount namespace copies the mount table, not the
+files behind it, so a private `resolv.conf` is a bind mount over a real file,
+which has to be created somewhere. The `scratch` volume is that somewhere. With
+a read-only root filesystem and no writable path, `nano-init` says so and stops.
+
+What the agent sees once this is running is what it would see in a microVM:
+
+```console
+$ kubectl exec -c agent agent -- ip -o link show | cut -d: -f2
+ lo
+ tun0
+```
+
+No `eth0`, on a pod that has one.
+
## Why this shape
The tempting alternative is to give the agent a token and a proxy and trust it
diff --git a/tests/integration/sandbox_namespaces_test.go b/tests/integration/sandbox_namespaces_test.go
new file mode 100644
index 00000000..9b8279e5
--- /dev/null
+++ b/tests/integration/sandbox_namespaces_test.go
@@ -0,0 +1,194 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration_test
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/google/sam/internal/sambox"
+)
+
+// TestSandboxCreatesItsOwnNamespaces covers the Kubernetes profile, which is
+// the one no runtime isolates.
+//
+// The other two profiles are handed a network namespace with nowhere to go:
+// `docker run --network none` makes one, and a microVM has its own kernel. A
+// pod makes neither -- every container in a pod shares one network namespace,
+// and the resolv.conf the kubelet writes is shared with them too -- so
+// nano-init has to build both for itself.
+//
+// This runs it the way a pod would: from a namespace that has real interfaces
+// and a real resolv.conf, with no external unshare and no privileges. What is
+// being asserted is that the agent ends up somewhere else entirely, and that
+// the boundary still reaches it there.
+//
+// It needs /dev/net/tun and unprivileged user namespaces, and skips without
+// them rather than passing quietly.
+func TestSandboxCreatesItsOwnNamespaces(t *testing.T) {
+ const (
+ allowedHost = "allowed.example"
+ blockedHost = "blocked.example"
+ serverBody = "reached the far side"
+ )
+
+ if _, err := os.Stat("/dev/net/tun"); err != nil {
+ t.Skipf("this profile needs /dev/net/tun, the same as a pod does: %v", err)
+ }
+
+ nanoInitBin := buildBinary(t, "./cmd/nano-init")
+
+ // The namespace this test runs in is emphatically not a sandbox, which is
+ // the point: it stands in for the pod's shared namespace. If nano-init
+ // used it, the assertions below would fail rather than quietly pass.
+ if !hasForeignInterfaces(t) {
+ t.Skip("this namespace has no interfaces, so it cannot stand in for a pod's")
+ }
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = io.WriteString(w, serverBody)
+ }))
+ defer server.Close()
+
+ serverURL, err := url.Parse(server.URL)
+ if err != nil {
+ t.Fatalf("parsing the server URL: %v", err)
+ }
+
+ sockDir, err := os.MkdirTemp("", "podsandbox")
+ if err != nil {
+ t.Fatalf("MkdirTemp: %v", err)
+ }
+ defer func() { _ = os.RemoveAll(sockDir) }()
+ agentSocket := filepath.Join(sockDir, "agent.sock")
+
+ var dialed struct {
+ sync.Mutex
+ names []string
+ }
+
+ egress, err := sambox.NewEgressPolicy([]string{allowedHost})
+ if err != nil {
+ t.Fatalf("NewEgressPolicy: %v", err)
+ }
+ listener, err := sambox.ListenSandboxSocket(agentSocket)
+ if err != nil {
+ t.Fatalf("ListenSandboxSocket: %v", err)
+ }
+
+ boundary := &sambox.SOCKS5Server{
+ Dialer: &sambox.AgentDialer{
+ Router: &sambox.Router{Egress: egress},
+ DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
+ host, _, err := net.SplitHostPort(address)
+ if err != nil {
+ host = address
+ }
+ dialed.Lock()
+ dialed.names = append(dialed.names, host)
+ dialed.Unlock()
+ return (&net.Dialer{}).DialContext(ctx, network, serverURL.Host)
+ },
+ },
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ served := make(chan struct{})
+ go func() {
+ defer close(served)
+ if err := boundary.Serve(ctx, listener); err != nil {
+ t.Errorf("boundary: %v", err)
+ }
+ }()
+ defer func() {
+ cancel()
+ <-served
+ }()
+
+ // The agent reports what it can see before using it, so a failure says
+ // whether the namespace was wrong or the datapath was.
+ agent := fmt.Sprintf(
+ `echo "links=$(ip -o link show | cut -d: -f2 | tr -d " " | paste -sd,)"; `+
+ `echo "resolv=$(cat /etc/resolv.conf)"; `+
+ `curl -sS --max-time 20 http://%s/; echo "allowed-exit=$?"; `+
+ `curl -sS --max-time 20 http://%s/; echo "blocked-exit=$?"`,
+ allowedHost, blockedHost,
+ )
+
+ runCtx, runCancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer runCancel()
+
+ out, err := exec.CommandContext(runCtx, nanoInitBin,
+ "run", "--create-namespaces", agentSocket, "bash", "-c", agent).CombinedOutput()
+ t.Logf("nano-init exited with %v, sandbox output:\n%s", err, out)
+
+ got := string(out)
+ if strings.Contains(got, "user namespaces are disabled") ||
+ strings.Contains(got, "unprivileged user namespaces are disabled") {
+ t.Skipf("this host does not allow the namespaces a pod profile needs:\n%s", got)
+ }
+
+ // The whole claim. A pod's namespace has eth0 and whatever else the CNI
+ // left; the agent must be somewhere with neither.
+ if !strings.Contains(got, "links=lo,tun0") {
+ t.Errorf("the agent did not get a namespace of its own; it reported:\n%s", got)
+ }
+ if !strings.Contains(got, "resolv=nameserver 169.254.1.1") {
+ t.Errorf("the agent kept the outer resolv.conf, so a pod's DNS would have been repointed:\n%s", got)
+ }
+
+ // And the boundary still reaches it, which is what makes the isolation
+ // useful rather than merely complete.
+ if !strings.Contains(got, serverBody) || !strings.Contains(got, "allowed-exit=0") {
+ t.Errorf("the sandbox did not reach %s through the boundary", allowedHost)
+ }
+ if strings.Contains(got, "blocked-exit=0") {
+ t.Errorf("reaching %s succeeded, want the boundary to refuse it", blockedHost)
+ }
+
+ dialed.Lock()
+ defer dialed.Unlock()
+ if len(dialed.names) != 1 || dialed.names[0] != allowedHost {
+ t.Errorf("the boundary dialled %q, want exactly [%q]", dialed.names, allowedHost)
+ }
+}
+
+// hasForeignInterfaces reports whether this namespace has anything but
+// loopback, which is what makes it a stand-in for a pod's.
+func hasForeignInterfaces(t *testing.T) bool {
+ t.Helper()
+ links, err := net.Interfaces()
+ if err != nil {
+ t.Fatalf("list interfaces: %v", err)
+ }
+ for _, l := range links {
+ if l.Name != "lo" {
+ return true
+ }
+ }
+ return false
+}