Skip to content

Agent Mesh

sandbox: run under GKE's default AppArmor with no securityContext

Antonio Ojea75fd4077eebd
diff --git a/.github/k8s/sam-box-canary-template.yaml b/.github/k8s/sam-box-canary-template.yaml
index 7d113382..d0ed183f 100644
--- a/.github/k8s/sam-box-canary-template.yaml
+++ b/.github/k8s/sam-box-canary-template.yaml
@@ -12,6 +12,19 @@ data:
rules: []
services: []
---
+# The resolver the sandbox uses, which is nano-init's own on the tun. Supplied
+# here so nano-init does not have to mount it: a pod's resolv.conf is shared by
+# every container, and replacing it from inside needs a bind mount that
+# containerd's default AppArmor profile denies.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sam-canary-sandbox-resolv-${ENV_NAME}
+ namespace: sam-canary-${ENV_NAME}
+data:
+ resolv.conf: |
+ nameserver 169.254.1.1
+---
apiVersion: v1
kind: ServiceAccount
metadata:
@@ -116,17 +129,13 @@ spec:
# uid: as root it would map 0 and hold no privilege over a file owned
# by 65532, and the connection would be refused.
#
- # Unconfined AppArmor is the one relaxation the sandbox needs. The
- # profile containerd applies by default permits the mount namespace and
- # then denies the bind mount inside it, so the sandbox would get a
- # namespace it cannot give a private resolv.conf. Everything else stays
- # off: no capabilities, no privileged, and a user namespace that holds
- # the process to what it owns.
+ # Nothing else is granted: no capabilities, not privileged, and no
+ # AppArmor change. The sandbox needs none because the resolv.conf it
+ # would otherwise have to bind-mount is mounted for it below, and that
+ # bind was the only thing containerd's default profile denied.
securityContext:
runAsUser: 65532
runAsGroup: 65532
- appArmorProfile:
- type: Unconfined
command:
- "/sandbox/nano-init"
- "run"
@@ -159,6 +168,14 @@ spec:
# has to be created somewhere.
- name: scratch
mountPath: /tmp
+ # The sandbox's own resolver, supplied by the kubelet rather than
+ # mounted by nano-init. A pod's resolv.conf is one file shared by every
+ # container in it, so the sandbox needs one of its own; having the
+ # kubelet mount it means nano-init never has to, which is what keeps
+ # this container free of any AppArmor exception.
+ - name: resolv
+ mountPath: /etc/resolv.conf
+ subPath: resolv.conf
# A bind mount is enough. The device cgroup does not deny this one, so
# there is no device plugin involved; what the tun needs is the
# capability, and the user namespace supplies that.
@@ -171,6 +188,9 @@ spec:
emptyDir: {}
- name: scratch
emptyDir: {}
+ - name: resolv
+ configMap:
+ name: sam-canary-sandbox-resolv-${ENV_NAME}
- name: tun
hostPath:
path: /dev/net/tun
diff --git a/cmd/nano-init/main.go b/cmd/nano-init/main.go
index 3a01ed17..6aa682a2 100644
--- a/cmd/nano-init/main.go
+++ b/cmd/nano-init/main.go
@@ -159,12 +159,16 @@ func run(createNS bool, ingressSocket, boundarySocket, cmdName string, cmdArgs [
if err != nil {
log.Fatalf("refusing to start: %v", err)
}
+ // Decided out here, where /etc/resolv.conf is the one the runtime gave
+ // this container: if it already names our resolver there is nothing to
+ // mount, and asking for a mount namespace would only invite a denial.
+ mountNS := !resolvConfAlreadyOurs()
self, err := os.Executable()
if err != nil {
log.Fatalf("locate this binary to re-execute it: %v", err)
}
args := runFlags(ingressSocket, boundarySocket, cmdName, cmdArgs)
- code, err := runAgent(ctx, cancel, self, args, withNamespaces(userNS))
+ code, err := runAgent(ctx, cancel, self, args, withNamespaces(userNS, mountNS))
if err != nil {
log.Fatalf("create the sandbox namespaces: %v\n%s", err, namespaceHint(err))
}
@@ -269,9 +273,13 @@ func setupNetwork(ctx context.Context, boundarySocket string, names *resolver) e
_ = dns.Close()
}()
- if err := os.WriteFile("/etc/resolv.conf", []byte("nameserver "+tunIP+"\n"), 0o644); err != nil {
- // Not fatal: resolution is a convenience here, not the control.
- log.Printf("could not write /etc/resolv.conf, name resolution may fail: %v", err)
+ // A pod can mount the file over instead, in which case it is read-only and
+ // already says this.
+ if !resolvConfAlreadyOurs() {
+ if err := os.WriteFile("/etc/resolv.conf", []byte("nameserver "+tunIP+"\n"), 0o644); err != nil {
+ // Not fatal: resolution is a convenience here, not the control.
+ log.Printf("could not write /etc/resolv.conf, name resolution may fail: %v", err)
+ }
}
// direct:// is a placeholder the engine insists on. The real dialer is
diff --git a/cmd/nano-init/namespaces.go b/cmd/nano-init/namespaces.go
index 3c59c0cd..eb3c191c 100644
--- a/cmd/nano-init/namespaces.go
+++ b/cmd/nano-init/namespaces.go
@@ -50,21 +50,31 @@ func insideCreatedNamespaces() bool {
return os.Getenv(nsCreatedEnv) == "1"
}
-// withNamespaces makes the child the first process in a new network and mount
-// namespace, adding a user namespace when that is the only way to be allowed.
+// withNamespaces makes the child the first process in a new network namespace,
+// adding a mount namespace when the sandbox needs one and a user namespace when
+// that is the only way to be allowed.
//
// The work happens in a child because unshare(CLONE_NEWNET) moves one thread,
// and the Go runtime has several that goroutines migrate between: the only way
// to get a whole program into a new network namespace is to start one there.
-func withNamespaces(userNS bool) func(*exec.Cmd) {
+//
+// The mount namespace is conditional because asking for one is not free. The
+// runtime marks / private when Unshareflags carries CLONE_NEWNS, and that mount
+// is exactly what containerd's default AppArmor profile denies -- so a sandbox
+// that was handed a resolv.conf of its own, and therefore has nothing to mount,
+// runs under that profile untouched by asking for no mount namespace at all.
+func withNamespaces(userNS, mountNS bool) func(*exec.Cmd) {
return func(c *exec.Cmd) {
if c.SysProcAttr == nil {
c.SysProcAttr = &syscall.SysProcAttr{}
}
c.SysProcAttr.Cloneflags |= syscall.CLONE_NEWNET
- // Unshareflags rather than Cloneflags: the runtime also makes / private
- // that way, so a bind mount below cannot propagate back to the pod.
- c.SysProcAttr.Unshareflags |= syscall.CLONE_NEWNS
+ if mountNS {
+ // Unshareflags rather than Cloneflags: the runtime also makes /
+ // private that way, so a bind mount below cannot propagate back to
+ // the pod.
+ c.SysProcAttr.Unshareflags |= syscall.CLONE_NEWNS
+ }
if userNS {
c.SysProcAttr.Cloneflags |= syscall.CLONE_NEWUSER
@@ -194,9 +204,18 @@ func mountHint(err error) string {
// without this, pointing DNS at the sandbox's own resolver would point every
// container in the pod at it.
//
+// Unless somebody already gave this container one of its own, which a pod can
+// do by mounting a file over /etc/resolv.conf. Then there is nothing to
+// arrange, and skipping it matters rather than merely saving work: the bind
+// below is the one operation containerd's default AppArmor profile denies, so
+// a sandbox that does not need it needs no profile change either.
+//
// Only the privacy is arranged here. What goes in the file is setupNetwork's
// business, and it writes through this mount.
func privateResolvConf() error {
+ if resolvConfAlreadyOurs() {
+ return nil
+ }
f, err := os.CreateTemp("", "resolv.conf")
if err != nil {
return fmt.Errorf(
@@ -215,3 +234,33 @@ func privateResolvConf() error {
_ = os.Remove(name)
return nil
}
+
+// resolvConfAlreadyOurs reports whether this sandbox has been handed a
+// resolv.conf naming the resolver it is about to run.
+//
+// Read rather than assumed: a file that names some other server would send the
+// agent's lookups somewhere outside the boundary, so only an exact match counts
+// as already done.
+func resolvConfAlreadyOurs() bool {
+ contents, err := os.ReadFile("/etc/resolv.conf")
+ if err != nil {
+ return false
+ }
+ return resolvConfSaysOurs(string(contents))
+}
+
+// resolvConfSaysOurs reports whether a resolv.conf names our resolver and
+// nothing else.
+func resolvConfSaysOurs(contents string) bool {
+ for _, line := range strings.Split(contents, "\n") {
+ fields := strings.Fields(line)
+ if len(fields) == 0 || strings.HasPrefix(fields[0], "#") {
+ continue
+ }
+ // Any nameserver but ours, and the file is not one we can rely on.
+ if !strings.EqualFold(fields[0], "nameserver") || len(fields) != 2 || fields[1] != tunIP {
+ return false
+ }
+ }
+ return strings.Contains(contents, tunIP)
+}
diff --git a/cmd/nano-init/namespaces_test.go b/cmd/nano-init/namespaces_test.go
index e79d860f..46c699bd 100644
--- a/cmd/nano-init/namespaces_test.go
+++ b/cmd/nano-init/namespaces_test.go
@@ -81,7 +81,7 @@ func TestCapSysAdminFromStatus(t *testing.T) {
func TestWithNamespaces(t *testing.T) {
t.Run("always makes a network and a mount namespace", func(t *testing.T) {
cmd := exec.Command("/bin/true")
- withNamespaces(false)(cmd)
+ withNamespaces(false, true)(cmd)
if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWNET == 0 {
t.Error("no CLONE_NEWNET: the sandbox would share the pod's network")
@@ -96,7 +96,7 @@ func TestWithNamespaces(t *testing.T) {
t.Run("adds a user namespace only when asked", func(t *testing.T) {
cmd := exec.Command("/bin/true")
- withNamespaces(false)(cmd)
+ withNamespaces(false, true)(cmd)
if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWUSER != 0 {
t.Error("CLONE_NEWUSER without being asked: it is a fallback, not the default")
@@ -108,7 +108,7 @@ func TestWithNamespaces(t *testing.T) {
t.Run("a user namespace comes with the mapping that makes it usable", func(t *testing.T) {
cmd := exec.Command("/bin/true")
- withNamespaces(true)(cmd)
+ withNamespaces(true, true)(cmd)
if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWUSER == 0 {
t.Fatal("no CLONE_NEWUSER: without CAP_SYS_ADMIN there is no other way to make a netns")
@@ -125,7 +125,7 @@ func TestWithNamespaces(t *testing.T) {
t.Run("marks the child so it does not do this again", func(t *testing.T) {
cmd := exec.Command("/bin/true")
- withNamespaces(false)(cmd)
+ withNamespaces(false, true)(cmd)
var found bool
for _, kv := range cmd.Env {
@@ -138,3 +138,51 @@ func TestWithNamespaces(t *testing.T) {
}
})
}
+
+func TestResolvConfAlreadyOurs(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ contents string
+ want bool
+ }{
+ {
+ name: "the file a pod can mount for the sandbox",
+ contents: "nameserver " + tunIP + "\n",
+ want: true,
+ },
+ {
+ name: "comments and blank lines are not nameservers",
+ contents: "# supplied by the platform\n\nnameserver " + tunIP + "\n",
+ want: true,
+ },
+ {
+ name: "a pod's own resolv.conf is not ours",
+ contents: "search default.svc.cluster.local svc.cluster.local\n" +
+ "nameserver 34.118.224.10\noptions ndots:5\n",
+ want: false,
+ },
+ {
+ // The dangerous one: ours plus somebody else's would send some
+ // lookups outside the boundary, so it does not count as done.
+ name: "ours alongside another server",
+ contents: "nameserver " + tunIP + "\nnameserver 8.8.8.8\n",
+ want: false,
+ },
+ {
+ name: "empty",
+ contents: "",
+ want: false,
+ },
+ {
+ name: "a nameserver line with no address",
+ contents: "nameserver\n",
+ want: false,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := resolvConfSaysOurs(tc.contents); got != tc.want {
+ t.Errorf("resolvConfSaysOurs(%q) = %v, want %v", tc.contents, got, tc.want)
+ }
+ })
+ }
+}
diff --git a/site/content/docs/user/running-agents.md b/site/content/docs/user/running-agents.md
index 900b18a4..470ea4d8 100644
--- a/site/content/docs/user/running-agents.md
+++ b/site/content/docs/user/running-agents.md
@@ -340,16 +340,30 @@ spec:
# starts, no network.
- name: agent
image: your-agent-image
- securityContext:
- # The one relaxation the sandbox needs; see below.
- appArmorProfile:
- type: Unconfined
command: ["/usr/local/bin/nano-init", "run", "--create-namespaces",
"/var/run/sam/agent.sock", "python3", "/app/agent.py"]
volumeMounts:
- { name: sam-uds, mountPath: /var/run/sam }
- { name: scratch, mountPath: /tmp }
- { name: tun, mountPath: /dev/net/tun }
+ # The sandbox's own resolver, supplied by the kubelet.
+ - { name: resolv, mountPath: /etc/resolv.conf, subPath: resolv.conf }
+```
+
+With the matching volume and a ConfigMap holding one line:
+
+```yaml
+ - name: resolv
+ configMap:
+ name: sandbox-resolv
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sandbox-resolv
+data:
+ resolv.conf: |
+ nameserver 169.254.1.1
```
Note what is **not** there: no `securityContext`, no added capabilities, no
@@ -369,12 +383,12 @@ holds both over the namespaces it then makes. Granting `CAP_SYS_ADMIN` alone is
in fact worse than granting nothing: the namespace gets created, and then the
tun fails for want of `CAP_NET_ADMIN`.
-**A seccomp and AppArmor policy that permits it.** This is the one that bites,
-and it is the reason for the `appArmorProfile` above. containerd applies an
-AppArmor profile by default — `cri-containerd.apparmor.d` on GKE — which permits
-creating the mount namespace and then **denies the bind mount inside it**. The
-sandbox gets a namespace it cannot put a private `resolv.conf` into. Measured on
-a GKE 1.35 node:
+**The sandbox needs a `resolv.conf` of its own, and the kubelet should supply
+it.** A pod's `resolv.conf` is one file shared by every container, so pointing
+DNS at the sandbox's resolver by writing that file would point `sam-node` at it
+too — at an address that only exists inside the sandbox's network namespace.
+`nano-init` can bind-mount a private one, but that bind is the single operation
+containerd's default AppArmor profile denies:
```console
user+net : ok
@@ -382,27 +396,32 @@ user+mount (unchanged propagation): ok
bind mount inside : FAILED
```
-With `appArmorProfile: {type: Unconfined}` on that container, all three pass.
-Seccomp is not involved: GKE reports `Seccomp: 0` for pods that do not ask for a
-profile, and creating the namespaces is allowed. Docker is the other way round —
-its default seccomp blocks the user namespace and its default AppArmor blocks the
-mount — which is why running the same image locally needs
-`--security-opt seccomp=unconfined --security-opt apparmor=unconfined`.
-
-It is worth being clear about what that costs, because the container it applies
-to is the least trusted one. The agent keeps every other constraint: no
-capabilities, not privileged, its own user namespace, and no network but the
-tun. What it gains is the ability to mount inside its own mount namespace, which
-a user namespace already confines to filesystems it owns. The tighter option is
-a custom profile — the default plus `mount` — loaded on the nodes and selected
-with `appArmorProfile: {type: Localhost, localhostProfile: ...}`; that is
-strictly better and costs an operational step, so it is the right thing to move
-to rather than the right thing to start with.
-
-**Somewhere writable.** A new mount namespace copies the mount table, not the
-files behind it, so a private `resolv.conf` is a bind mount over a real file,
-which has to be created somewhere. The `scratch` volume is that somewhere. With
-a read-only root filesystem and no writable path, `nano-init` says so and stops.
+Mounting the file per container avoids the question. `nano-init` checks whether
+`/etc/resolv.conf` already names its resolver and, if so, does nothing — so the
+container needs no `securityContext` at all: no capabilities, not privileged, no
+AppArmor exception.
+
+**Why not `dnsConfig`?** Because `dnsPolicy` and `dnsConfig` are PodSpec fields
+and apply to every container in the pod. Measured on GKE, the per-container
+mount gives you this:
+
+```console
+[sandbox] nameserver 169.254.1.1
+[node] search default.svc.cluster.local ... nameserver 34.118.224.10
+[node] cluster DNS works
+```
+
+With `dnsConfig`, `sam-node` would get `169.254.1.1` as well and could no longer
+resolve the control plane.
+
+**If you do not supply one, it needs somewhere writable.** Without the mount
+above, `nano-init` falls back to bind-mounting a private `resolv.conf` itself,
+which needs a real file to bind and therefore a writable path — an `emptyDir` on
+`/tmp` will do. That path also needs a mount namespace, and the bind is what
+AppArmor refuses, so on a default GKE node the fallback fails and the supplied
+file is the way through. Verified on GKE 1.35: with the file mounted, the
+sandbox starts under `cri-containerd.apparmor.d` with no `securityContext`;
+without it, `nano-init` reports `permission denied` creating the namespaces.
### The harness must be nano-init's child
diff --git a/tests/e2e/canary_manifests.bats b/tests/e2e/canary_manifests.bats
index 9e018d29..c7cb43f5 100644
--- a/tests/e2e/canary_manifests.bats
+++ b/tests/e2e/canary_manifests.bats
@@ -102,6 +102,26 @@ unservable() {
[[ "${failed}" -eq 0 ]]
}
+@test "Canary manifests: the sandbox resolv.conf names nano-init's own resolver" {
+ # The canary hands the sandbox a resolv.conf so nano-init does not have to
+ # mount one, and that only works while the address in it is the address
+ # nano-init answers on. A mismatch is quiet in the wrong way: nano-init falls
+ # back to the bind mount, which kind permits and containerd's default AppArmor
+ # profile denies, so it would pass here and fail on the cluster.
+ local declared resolver
+ declared=$(grep -A2 'resolv.conf: |' .github/k8s/sam-box-canary-template.yaml \
+ | grep nameserver | awk '{print $2}')
+ resolver=$(grep -oE 'tunIP[[:space:]]*=[[:space:]]*"[0-9.]+"' cmd/nano-init/main.go \
+ | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+')
+
+ echo "# canary declares: ${declared}" >&3
+ echo "# nano-init serves: ${resolver}" >&3
+
+ [[ -n "${declared}" ]]
+ [[ -n "${resolver}" ]]
+ [[ "${declared}" == "${resolver}" ]]
+}
+
@test "Canary rollout: the sandbox canary comes up and is a sandbox" {
# deploy.yaml pulls these from ghcr; here they are the images this commit
# just built, retagged so the manifest under test is the manifest shipped