Skip to content

Agent Mesh

nano-init: create the sandbox where no runtime will

Antonio Ojeab62404dd005b
diff --git a/cmd/nano-init/main.go b/cmd/nano-init/main.go
index 0cf4886c..9a43258b 100644
--- a/cmd/nano-init/main.go
+++ b/cmd/nano-init/main.go
@@ -95,10 +95,15 @@ func main() {
}
case "run":
- if len(os.Args) < 4 {
- log.Fatalf("usage: %s run <boundary-socket> <cmd> [args...]", os.Args[0])
+ args := os.Args[2:]
+ createNS := false
+ if len(args) > 0 && args[0] == "--create-namespaces" {
+ createNS, args = true, args[1:]
}
- run(os.Args[2], os.Args[3], os.Args[4:])
+ if len(args) < 2 {
+ log.Fatalf("usage: %s run [--create-namespaces] <boundary-socket> <cmd> [args...]", os.Args[0])
+ }
+ run(createNS, args[0], args[1], args[2:])
default:
usage()
@@ -106,15 +111,38 @@ func main() {
}
func usage() {
- log.Fatalf("usage:\n %s copy <dest>\n %s run <boundary-socket> <cmd> [args...]",
+ log.Fatalf("usage:\n %s copy <dest>\n %s run [--create-namespaces] <boundary-socket> <cmd> [args...]",
os.Args[0], os.Args[0])
}
// run wires the sandbox up and hands it to the agent.
-func run(boundarySocket, cmdName string, cmdArgs []string) {
+func run(createNS bool, boundarySocket, cmdName string, cmdArgs []string) {
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
defer cancel()
+ // The namespaces have to exist before anything is checked in them, and a
+ // whole Go program can only enter a new network namespace by being started
+ // in one. So this half makes them and becomes a supervisor; the half that
+ // comes back through here does the work.
+ if createNS && !insideCreatedNamespaces() {
+ userNS, err := needUserNamespace()
+ if err != nil {
+ log.Fatalf("refusing to start: %v", err)
+ }
+ self, err := os.Executable()
+ if err != nil {
+ log.Fatalf("locate this binary to re-execute it: %v", err)
+ }
+ args := append([]string{"run", "--create-namespaces", boundarySocket, cmdName}, cmdArgs...)
+ os.Exit(runAgent(ctx, cancel, self, args, withNamespaces(userNS)))
+ }
+
+ if createNS {
+ if err := privateResolvConf(); err != nil {
+ log.Fatalf("refusing to start: %v", err)
+ }
+ }
+
// First, and before anything is built: if this namespace is not a sandbox
// then the boundary is beside the point, and saying so in that order is
// the difference between "you forgot --network none" and a puzzling
@@ -156,11 +184,7 @@ func setupNetwork(ctx context.Context, boundarySocket string, names *resolver) e
Mode: netlink.TUNTAP_MODE_TUN,
}
if err := netlink.LinkAdd(tun); err != nil {
- // The usual cause is a guest kernel built without CONFIG_TUN, which is
- // the case for the stock Firecracker CI kernels: they carry vsock but
- // no tun driver. Without it there is no way to give the sandbox a
- // route, so this is fatal rather than something to work around.
- return fmt.Errorf("create %s (does this kernel have CONFIG_TUN?): %w", tunName, err)
+ return fmt.Errorf("create %s: %w\n%s", tunName, err, describeTunFailure(err))
}
if err := netlink.LinkSetUp(tun); err != nil {
return fmt.Errorf("bring up %s: %w", tunName, err)
@@ -216,11 +240,18 @@ func setupNetwork(ctx context.Context, boundarySocket string, names *resolver) e
}
// runAgent starts the agent and reports the exit status it should be judged by.
-func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cmdArgs []string) int {
+//
+// The same supervision serves the namespace trampoline, whose child is this
+// binary again: orphans still reparent here and still have to be reaped, and
+// the exit code still has to be the one the caller sees.
+func runAgent(ctx context.Context, cancel context.CancelFunc, cmdName string, cmdArgs []string, opts ...func(*exec.Cmd)) int {
cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
cmd.Stdout, cmd.Stderr, cmd.Stdin = os.Stdout, os.Stderr, os.Stdin
cmd.Env = os.Environ() // Nothing injected: the agent is not configured, it is routed.
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ for _, opt := range opts {
+ opt(cmd)
+ }
cmd.Cancel = func() error {
if cmd.Process == nil {
return nil
diff --git a/cmd/nano-init/namespaces.go b/cmd/nano-init/namespaces.go
new file mode 100644
index 00000000..b493f9c1
--- /dev/null
+++ b/cmd/nano-init/namespaces.go
@@ -0,0 +1,178 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "strconv"
+ "strings"
+ "syscall"
+
+ "golang.org/x/sys/unix"
+)
+
+// Creating the sandbox is normally somebody else's job: `docker run
+// --network none` and a microVM's own kernel both hand this process a
+// namespace with nowhere to go. A Kubernetes pod does not, and cannot -- every
+// container in a pod shares one network namespace, and the resolv.conf the
+// kubelet writes is shared with them too -- so for that profile the namespaces
+// have to be made here, from inside the container.
+//
+// This is opt-in rather than automatic. A sandbox that quietly creates its own
+// isolation when it cannot find any is a sandbox that never reports a
+// misconfigured runtime, and the two profiles that do get isolation from their
+// runtime should keep failing loudly when it is missing.
+
+// nsCreatedEnv marks the re-executed child, so it sets the namespaces up once
+// rather than forever.
+const nsCreatedEnv = "NANO_INIT_NAMESPACES_CREATED"
+
+// capSysAdmin is the capability that creating a network namespace requires.
+const capSysAdmin = 21
+
+// insideCreatedNamespaces reports whether this process is the re-executed half.
+func insideCreatedNamespaces() bool {
+ return os.Getenv(nsCreatedEnv) == "1"
+}
+
+// withNamespaces makes the child the first process in a new network and mount
+// namespace, adding a user namespace when that is the only way to be allowed.
+//
+// The work happens in a child because unshare(CLONE_NEWNET) moves one thread,
+// and the Go runtime has several that goroutines migrate between: the only way
+// to get a whole program into a new network namespace is to start one there.
+func withNamespaces(userNS bool) func(*exec.Cmd) {
+ return func(c *exec.Cmd) {
+ if c.SysProcAttr == nil {
+ c.SysProcAttr = &syscall.SysProcAttr{}
+ }
+ c.SysProcAttr.Cloneflags |= syscall.CLONE_NEWNET
+ // Unshareflags rather than Cloneflags: the runtime also makes / private
+ // that way, so a bind mount below cannot propagate back to the pod.
+ c.SysProcAttr.Unshareflags |= syscall.CLONE_NEWNS
+
+ if userNS {
+ c.SysProcAttr.Cloneflags |= syscall.CLONE_NEWUSER
+ c.SysProcAttr.UidMappings = []syscall.SysProcIDMap{
+ {ContainerID: 0, HostID: os.Getuid(), Size: 1},
+ }
+ c.SysProcAttr.GidMappings = []syscall.SysProcIDMap{
+ {ContainerID: 0, HostID: os.Getgid(), Size: 1},
+ }
+ // Denied because an unprivileged user namespace may not call it,
+ // and nothing in a sandbox needs supplementary groups.
+ c.SysProcAttr.GidMappingsEnableSetgroups = false
+ }
+
+ c.Env = append(c.Env, nsCreatedEnv+"=1")
+ }
+}
+
+// needUserNamespace decides how to get permission to create a network
+// namespace, or explains why neither way is open.
+//
+// A user namespace is not the goal, it is the fallback: inside one this process
+// is root and holds CAP_SYS_ADMIN over the namespaces it then creates, which is
+// what makes the pod profile work without the pod being privileged. When
+// CAP_SYS_ADMIN is already held there is no reason to add the extra namespace
+// and the uid mapping that comes with it.
+func needUserNamespace() (bool, error) {
+ switch has, err := hasCapSysAdmin(); {
+ case err != nil:
+ return false, err
+ case has:
+ return false, nil
+ }
+ if err := userNamespacesAvailable(); err != nil {
+ return false, fmt.Errorf(
+ "cannot create a network namespace: this process has no CAP_SYS_ADMIN, and %w. "+
+ "Grant CAP_SYS_ADMIN, or allow unprivileged user namespaces", err)
+ }
+ return true, nil
+}
+
+// hasCapSysAdmin reads the effective capability set of this process.
+func hasCapSysAdmin() (bool, error) {
+ status, err := os.ReadFile("/proc/self/status")
+ if err != nil {
+ return false, fmt.Errorf("read capabilities: %w", err)
+ }
+ return capSysAdminFromStatus(string(status))
+}
+
+// capSysAdminFromStatus finds CAP_SYS_ADMIN in the CapEff line of a
+// /proc/<pid>/status.
+func capSysAdminFromStatus(status string) (bool, error) {
+ for _, line := range strings.Split(status, "\n") {
+ hex, ok := strings.CutPrefix(line, "CapEff:")
+ if !ok {
+ continue
+ }
+ caps, err := strconv.ParseUint(strings.TrimSpace(hex), 16, 64)
+ if err != nil {
+ return false, fmt.Errorf("parse CapEff %q: %w", strings.TrimSpace(hex), err)
+ }
+ return caps&(1<<capSysAdmin) != 0, nil
+ }
+ return false, fmt.Errorf("no CapEff line in /proc/self/status")
+}
+
+// userNamespacesAvailable reports why an unprivileged user namespace would be
+// refused, so the failure names a sysctl rather than an errno.
+func userNamespacesAvailable() error {
+ if max, err := os.ReadFile("/proc/sys/user/max_user_namespaces"); err == nil {
+ if n, err := strconv.Atoi(strings.TrimSpace(string(max))); err == nil && n == 0 {
+ return fmt.Errorf("user namespaces are disabled (user.max_user_namespaces is 0)")
+ }
+ }
+ // Debian and derivatives carry this one separately.
+ if allowed, err := os.ReadFile("/proc/sys/kernel/unprivileged_userns_clone"); err == nil {
+ if strings.TrimSpace(string(allowed)) == "0" {
+ return fmt.Errorf("unprivileged user namespaces are disabled (kernel.unprivileged_userns_clone is 0)")
+ }
+ }
+ return nil
+}
+
+// privateResolvConf gives the sandbox a resolv.conf of its own.
+//
+// A new mount namespace copies the mount table; it does not copy files. The
+// resolv.conf a pod hands each container is one file shared by all of them, so
+// without this, pointing DNS at the sandbox's own resolver would point every
+// container in the pod at it.
+//
+// Only the privacy is arranged here. What goes in the file is setupNetwork's
+// business, and it writes through this mount.
+func privateResolvConf() error {
+ f, err := os.CreateTemp("", "resolv.conf")
+ if err != nil {
+ return fmt.Errorf(
+ "a private resolv.conf needs somewhere writable to live, and this sandbox has nowhere: %w. "+
+ "Give the container a writable /tmp, or an emptyDir mounted there", err)
+ }
+ name := f.Name()
+ if err := f.Close(); err != nil {
+ return fmt.Errorf("close the private resolv.conf: %w", err)
+ }
+ if err := unix.Mount(name, "/etc/resolv.conf", "", unix.MS_BIND, ""); err != nil {
+ _ = os.Remove(name)
+ return fmt.Errorf("bind a private resolv.conf over /etc/resolv.conf: %w", err)
+ }
+ // The mount holds the inode, so the path it was reached by is litter.
+ _ = os.Remove(name)
+ return nil
+}
diff --git a/cmd/nano-init/namespaces_test.go b/cmd/nano-init/namespaces_test.go
new file mode 100644
index 00000000..e79d860f
--- /dev/null
+++ b/cmd/nano-init/namespaces_test.go
@@ -0,0 +1,140 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "os/exec"
+ "syscall"
+ "testing"
+)
+
+func TestCapSysAdminFromStatus(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ status string
+ want bool
+ wantErr bool
+ }{
+ {
+ name: "a full capability set has it",
+ status: "Name:\tnano-init\nCapEff:\t000001ffffffffff\n",
+ want: true,
+ },
+ {
+ name: "a pod's default set does not",
+ // What a container gets without securityContext: no CAP_SYS_ADMIN,
+ // which is the case that has to fall back to a user namespace.
+ status: "Name:\tnano-init\nCapEff:\t00000000a80425fb\n",
+ want: false,
+ },
+ {
+ name: "an empty set does not",
+ status: "CapEff:\t0000000000000000\n",
+ want: false,
+ },
+ {
+ name: "exactly CAP_SYS_ADMIN and nothing else",
+ status: "CapEff:\t0000000000200000\n",
+ want: true,
+ },
+ {
+ name: "a status with no CapEff line is an error, not a false",
+ status: "Name:\tnano-init\nPid:\t1\n",
+ wantErr: true,
+ },
+ {
+ name: "an unparseable set is an error, not a false",
+ status: "CapEff:\tnonsense\n",
+ wantErr: true,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := capSysAdminFromStatus(tc.status)
+ if tc.wantErr {
+ if err == nil {
+ t.Fatalf("capSysAdminFromStatus(%q) = %v, want an error: guessing here decides whether a sandbox is isolated", tc.status, got)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("capSysAdminFromStatus(%q): %v", tc.status, err)
+ }
+ if got != tc.want {
+ t.Errorf("capSysAdminFromStatus(%q) = %v, want %v", tc.status, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestWithNamespaces(t *testing.T) {
+ t.Run("always makes a network and a mount namespace", func(t *testing.T) {
+ cmd := exec.Command("/bin/true")
+ withNamespaces(false)(cmd)
+
+ if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWNET == 0 {
+ t.Error("no CLONE_NEWNET: the sandbox would share the pod's network")
+ }
+ // Unshareflags rather than Cloneflags, because that is what makes the
+ // runtime mark / private; without it the bind mount over resolv.conf
+ // could propagate back to the pod.
+ if cmd.SysProcAttr.Unshareflags&syscall.CLONE_NEWNS == 0 {
+ t.Error("no CLONE_NEWNS in Unshareflags: mounts could escape the sandbox")
+ }
+ })
+
+ t.Run("adds a user namespace only when asked", func(t *testing.T) {
+ cmd := exec.Command("/bin/true")
+ withNamespaces(false)(cmd)
+
+ if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWUSER != 0 {
+ t.Error("CLONE_NEWUSER without being asked: it is a fallback, not the default")
+ }
+ if len(cmd.SysProcAttr.UidMappings) != 0 {
+ t.Error("uid mappings without a user namespace to map into")
+ }
+ })
+
+ t.Run("a user namespace comes with the mapping that makes it usable", func(t *testing.T) {
+ cmd := exec.Command("/bin/true")
+ withNamespaces(true)(cmd)
+
+ if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWUSER == 0 {
+ t.Fatal("no CLONE_NEWUSER: without CAP_SYS_ADMIN there is no other way to make a netns")
+ }
+ if len(cmd.SysProcAttr.UidMappings) != 1 || cmd.SysProcAttr.UidMappings[0].ContainerID != 0 {
+ t.Errorf("uid mappings = %v, want this process mapped to root inside", cmd.SysProcAttr.UidMappings)
+ }
+ // An unprivileged user namespace may not call setgroups, so asking for
+ // it fails the whole clone.
+ if cmd.SysProcAttr.GidMappingsEnableSetgroups {
+ t.Error("setgroups enabled: an unprivileged user namespace cannot have it")
+ }
+ })
+
+ t.Run("marks the child so it does not do this again", func(t *testing.T) {
+ cmd := exec.Command("/bin/true")
+ withNamespaces(false)(cmd)
+
+ var found bool
+ for _, kv := range cmd.Env {
+ if kv == nsCreatedEnv+"=1" {
+ found = true
+ }
+ }
+ if !found {
+ t.Errorf("env = %v, want %s: without it the child re-executes forever", cmd.Env, nsCreatedEnv)
+ }
+ })
+}