Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
sandbox: run under GKE's default AppArmor with no securityContext
Antonio Ojea75fd4077eebd
diff --git a/.github/k8s/sam-box-canary-template.yaml b/.github/k8s/sam-box-canary-template.yamlindex 7d113382..d0ed183f 100644--- a/.github/k8s/sam-box-canary-template.yaml+++ b/.github/k8s/sam-box-canary-template.yaml@@ -12,6 +12,19 @@ data:rules: []services: []---+# The resolver the sandbox uses, which is nano-init's own on the tun. Supplied+# here so nano-init does not have to mount it: a pod's resolv.conf is shared by+# every container, and replacing it from inside needs a bind mount that+# containerd's default AppArmor profile denies.+apiVersion: v1+kind: ConfigMap+metadata:+ name: sam-canary-sandbox-resolv-${ENV_NAME}+ namespace: sam-canary-${ENV_NAME}+data:+ resolv.conf: |+ nameserver 169.254.1.1+---apiVersion: v1kind: ServiceAccountmetadata:@@ -116,17 +129,13 @@ spec:# uid: as root it would map 0 and hold no privilege over a file owned# by 65532, and the connection would be refused.#- # Unconfined AppArmor is the one relaxation the sandbox needs. The- # profile containerd applies by default permits the mount namespace and- # then denies the bind mount inside it, so the sandbox would get a- # namespace it cannot give a private resolv.conf. Everything else stays- # off: no capabilities, no privileged, and a user namespace that holds- # the process to what it owns.+ # Nothing else is granted: no capabilities, not privileged, and no+ # AppArmor change. The sandbox needs none because the resolv.conf it+ # would otherwise have to bind-mount is mounted for it below, and that+ # bind was the only thing containerd's default profile denied.securityContext:runAsUser: 65532runAsGroup: 65532- appArmorProfile:- type: Unconfinedcommand:- "/sandbox/nano-init"- "run"@@ -159,6 +168,14 @@ spec:# has to be created somewhere.- name: scratchmountPath: /tmp+ # The sandbox's own resolver, supplied by the kubelet rather than+ # mounted by nano-init. A pod's resolv.conf is one file shared by every+ # container in it, so the sandbox needs one of its own; having the+ # kubelet mount it means nano-init never has to, which is what keeps+ # this container free of any AppArmor exception.+ - name: resolv+ mountPath: /etc/resolv.conf+ subPath: resolv.conf# A bind mount is enough. The device cgroup does not deny this one, so# there is no device plugin involved; what the tun needs is the# capability, and the user namespace supplies that.@@ -171,6 +188,9 @@ spec:emptyDir: {}- name: scratchemptyDir: {}+ - name: resolv+ configMap:+ name: sam-canary-sandbox-resolv-${ENV_NAME}- name: tunhostPath:path: /dev/net/tundiff --git a/cmd/nano-init/main.go b/cmd/nano-init/main.goindex 3a01ed17..6aa682a2 100644--- a/cmd/nano-init/main.go+++ b/cmd/nano-init/main.go@@ -159,12 +159,16 @@ func run(createNS bool, ingressSocket, boundarySocket, cmdName string, cmdArgs [if err != nil {log.Fatalf("refusing to start: %v", err)}+ // Decided out here, where /etc/resolv.conf is the one the runtime gave+ // this container: if it already names our resolver there is nothing to+ // mount, and asking for a mount namespace would only invite a denial.+ mountNS := !resolvConfAlreadyOurs()self, err := os.Executable()if err != nil {log.Fatalf("locate this binary to re-execute it: %v", err)}args := runFlags(ingressSocket, boundarySocket, cmdName, cmdArgs)- code, err := runAgent(ctx, cancel, self, args, withNamespaces(userNS))+ code, err := runAgent(ctx, cancel, self, args, withNamespaces(userNS, mountNS))if err != nil {log.Fatalf("create the sandbox namespaces: %v\n%s", err, namespaceHint(err))}@@ -269,9 +273,13 @@ func setupNetwork(ctx context.Context, boundarySocket string, names *resolver) e_ = dns.Close()}()- if err := os.WriteFile("/etc/resolv.conf", []byte("nameserver "+tunIP+"\n"), 0o644); err != nil {- // Not fatal: resolution is a convenience here, not the control.- log.Printf("could not write /etc/resolv.conf, name resolution may fail: %v", err)+ // A pod can mount the file over instead, in which case it is read-only and+ // already says this.+ if !resolvConfAlreadyOurs() {+ if err := os.WriteFile("/etc/resolv.conf", []byte("nameserver "+tunIP+"\n"), 0o644); err != nil {+ // Not fatal: resolution is a convenience here, not the control.+ log.Printf("could not write /etc/resolv.conf, name resolution may fail: %v", err)+ }}// direct:// is a placeholder the engine insists on. The real dialer isdiff --git a/cmd/nano-init/namespaces.go b/cmd/nano-init/namespaces.goindex 3c59c0cd..eb3c191c 100644--- a/cmd/nano-init/namespaces.go+++ b/cmd/nano-init/namespaces.go@@ -50,21 +50,31 @@ func insideCreatedNamespaces() bool {return os.Getenv(nsCreatedEnv) == "1"}-// withNamespaces makes the child the first process in a new network and mount-// namespace, adding a user namespace when that is the only way to be allowed.+// withNamespaces makes the child the first process in a new network namespace,+// adding a mount namespace when the sandbox needs one and a user namespace when+// that is the only way to be allowed.//// The work happens in a child because unshare(CLONE_NEWNET) moves one thread,// and the Go runtime has several that goroutines migrate between: the only way// to get a whole program into a new network namespace is to start one there.-func withNamespaces(userNS bool) func(*exec.Cmd) {+//+// The mount namespace is conditional because asking for one is not free. The+// runtime marks / private when Unshareflags carries CLONE_NEWNS, and that mount+// is exactly what containerd's default AppArmor profile denies -- so a sandbox+// that was handed a resolv.conf of its own, and therefore has nothing to mount,+// runs under that profile untouched by asking for no mount namespace at all.+func withNamespaces(userNS, mountNS bool) func(*exec.Cmd) {return func(c *exec.Cmd) {if c.SysProcAttr == nil {c.SysProcAttr = &syscall.SysProcAttr{}}c.SysProcAttr.Cloneflags |= syscall.CLONE_NEWNET- // Unshareflags rather than Cloneflags: the runtime also makes / private- // that way, so a bind mount below cannot propagate back to the pod.- c.SysProcAttr.Unshareflags |= syscall.CLONE_NEWNS+ if mountNS {+ // Unshareflags rather than Cloneflags: the runtime also makes /+ // private that way, so a bind mount below cannot propagate back to+ // the pod.+ c.SysProcAttr.Unshareflags |= syscall.CLONE_NEWNS+ }if userNS {c.SysProcAttr.Cloneflags |= syscall.CLONE_NEWUSER@@ -194,9 +204,18 @@ func mountHint(err error) string {// without this, pointing DNS at the sandbox's own resolver would point every// container in the pod at it.//+// Unless somebody already gave this container one of its own, which a pod can+// do by mounting a file over /etc/resolv.conf. Then there is nothing to+// arrange, and skipping it matters rather than merely saving work: the bind+// below is the one operation containerd's default AppArmor profile denies, so+// a sandbox that does not need it needs no profile change either.+//// Only the privacy is arranged here. What goes in the file is setupNetwork's// business, and it writes through this mount.func privateResolvConf() error {+ if resolvConfAlreadyOurs() {+ return nil+ }f, err := os.CreateTemp("", "resolv.conf")if err != nil {return fmt.Errorf(@@ -215,3 +234,33 @@ func privateResolvConf() error {_ = os.Remove(name)return nil}++// resolvConfAlreadyOurs reports whether this sandbox has been handed a+// resolv.conf naming the resolver it is about to run.+//+// Read rather than assumed: a file that names some other server would send the+// agent's lookups somewhere outside the boundary, so only an exact match counts+// as already done.+func resolvConfAlreadyOurs() bool {+ contents, err := os.ReadFile("/etc/resolv.conf")+ if err != nil {+ return false+ }+ return resolvConfSaysOurs(string(contents))+}++// resolvConfSaysOurs reports whether a resolv.conf names our resolver and+// nothing else.+func resolvConfSaysOurs(contents string) bool {+ for _, line := range strings.Split(contents, "\n") {+ fields := strings.Fields(line)+ if len(fields) == 0 || strings.HasPrefix(fields[0], "#") {+ continue+ }+ // Any nameserver but ours, and the file is not one we can rely on.+ if !strings.EqualFold(fields[0], "nameserver") || len(fields) != 2 || fields[1] != tunIP {+ return false+ }+ }+ return strings.Contains(contents, tunIP)+}diff --git a/cmd/nano-init/namespaces_test.go b/cmd/nano-init/namespaces_test.goindex e79d860f..46c699bd 100644--- a/cmd/nano-init/namespaces_test.go+++ b/cmd/nano-init/namespaces_test.go@@ -81,7 +81,7 @@ func TestCapSysAdminFromStatus(t *testing.T) {func TestWithNamespaces(t *testing.T) {t.Run("always makes a network and a mount namespace", func(t *testing.T) {cmd := exec.Command("/bin/true")- withNamespaces(false)(cmd)+ withNamespaces(false, true)(cmd)if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWNET == 0 {t.Error("no CLONE_NEWNET: the sandbox would share the pod's network")@@ -96,7 +96,7 @@ func TestWithNamespaces(t *testing.T) {t.Run("adds a user namespace only when asked", func(t *testing.T) {cmd := exec.Command("/bin/true")- withNamespaces(false)(cmd)+ withNamespaces(false, true)(cmd)if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWUSER != 0 {t.Error("CLONE_NEWUSER without being asked: it is a fallback, not the default")@@ -108,7 +108,7 @@ func TestWithNamespaces(t *testing.T) {t.Run("a user namespace comes with the mapping that makes it usable", func(t *testing.T) {cmd := exec.Command("/bin/true")- withNamespaces(true)(cmd)+ withNamespaces(true, true)(cmd)if cmd.SysProcAttr.Cloneflags&syscall.CLONE_NEWUSER == 0 {t.Fatal("no CLONE_NEWUSER: without CAP_SYS_ADMIN there is no other way to make a netns")@@ -125,7 +125,7 @@ func TestWithNamespaces(t *testing.T) {t.Run("marks the child so it does not do this again", func(t *testing.T) {cmd := exec.Command("/bin/true")- withNamespaces(false)(cmd)+ withNamespaces(false, true)(cmd)var found boolfor _, kv := range cmd.Env {@@ -138,3 +138,51 @@ func TestWithNamespaces(t *testing.T) {}})}++func TestResolvConfAlreadyOurs(t *testing.T) {+ for _, tc := range []struct {+ name string+ contents string+ want bool+ }{+ {+ name: "the file a pod can mount for the sandbox",+ contents: "nameserver " + tunIP + "\n",+ want: true,+ },+ {+ name: "comments and blank lines are not nameservers",+ contents: "# supplied by the platform\n\nnameserver " + tunIP + "\n",+ want: true,+ },+ {+ name: "a pod's own resolv.conf is not ours",+ contents: "search default.svc.cluster.local svc.cluster.local\n" ++ "nameserver 34.118.224.10\noptions ndots:5\n",+ want: false,+ },+ {+ // The dangerous one: ours plus somebody else's would send some+ // lookups outside the boundary, so it does not count as done.+ name: "ours alongside another server",+ contents: "nameserver " + tunIP + "\nnameserver 8.8.8.8\n",+ want: false,+ },+ {+ name: "empty",+ contents: "",+ want: false,+ },+ {+ name: "a nameserver line with no address",+ contents: "nameserver\n",+ want: false,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ if got := resolvConfSaysOurs(tc.contents); got != tc.want {+ t.Errorf("resolvConfSaysOurs(%q) = %v, want %v", tc.contents, got, tc.want)+ }+ })+ }+}diff --git a/site/content/docs/user/running-agents.md b/site/content/docs/user/running-agents.mdindex 900b18a4..470ea4d8 100644--- a/site/content/docs/user/running-agents.md+++ b/site/content/docs/user/running-agents.md@@ -340,16 +340,30 @@ spec:# starts, no network.- name: agentimage: your-agent-image- securityContext:- # The one relaxation the sandbox needs; see below.- appArmorProfile:- type: Unconfinedcommand: ["/usr/local/bin/nano-init", "run", "--create-namespaces","/var/run/sam/agent.sock", "python3", "/app/agent.py"]volumeMounts:- { name: sam-uds, mountPath: /var/run/sam }- { name: scratch, mountPath: /tmp }- { name: tun, mountPath: /dev/net/tun }+ # The sandbox's own resolver, supplied by the kubelet.+ - { name: resolv, mountPath: /etc/resolv.conf, subPath: resolv.conf }+```++With the matching volume and a ConfigMap holding one line:++```yaml+ - name: resolv+ configMap:+ name: sandbox-resolv+---+apiVersion: v1+kind: ConfigMap+metadata:+ name: sandbox-resolv+data:+ resolv.conf: |+ nameserver 169.254.1.1```Note what is **not** there: no `securityContext`, no added capabilities, no@@ -369,12 +383,12 @@ holds both over the namespaces it then makes. Granting `CAP_SYS_ADMIN` alone isin fact worse than granting nothing: the namespace gets created, and then thetun fails for want of `CAP_NET_ADMIN`.-**A seccomp and AppArmor policy that permits it.** This is the one that bites,-and it is the reason for the `appArmorProfile` above. containerd applies an-AppArmor profile by default — `cri-containerd.apparmor.d` on GKE — which permits-creating the mount namespace and then **denies the bind mount inside it**. The-sandbox gets a namespace it cannot put a private `resolv.conf` into. Measured on-a GKE 1.35 node:+**The sandbox needs a `resolv.conf` of its own, and the kubelet should supply+it.** A pod's `resolv.conf` is one file shared by every container, so pointing+DNS at the sandbox's resolver by writing that file would point `sam-node` at it+too — at an address that only exists inside the sandbox's network namespace.+`nano-init` can bind-mount a private one, but that bind is the single operation+containerd's default AppArmor profile denies:```consoleuser+net : ok@@ -382,27 +396,32 @@ user+mount (unchanged propagation): okbind mount inside : FAILED```-With `appArmorProfile: {type: Unconfined}` on that container, all three pass.-Seccomp is not involved: GKE reports `Seccomp: 0` for pods that do not ask for a-profile, and creating the namespaces is allowed. Docker is the other way round —-its default seccomp blocks the user namespace and its default AppArmor blocks the-mount — which is why running the same image locally needs-`--security-opt seccomp=unconfined --security-opt apparmor=unconfined`.--It is worth being clear about what that costs, because the container it applies-to is the least trusted one. The agent keeps every other constraint: no-capabilities, not privileged, its own user namespace, and no network but the-tun. What it gains is the ability to mount inside its own mount namespace, which-a user namespace already confines to filesystems it owns. The tighter option is-a custom profile — the default plus `mount` — loaded on the nodes and selected-with `appArmorProfile: {type: Localhost, localhostProfile: ...}`; that is-strictly better and costs an operational step, so it is the right thing to move-to rather than the right thing to start with.--**Somewhere writable.** A new mount namespace copies the mount table, not the-files behind it, so a private `resolv.conf` is a bind mount over a real file,-which has to be created somewhere. The `scratch` volume is that somewhere. With-a read-only root filesystem and no writable path, `nano-init` says so and stops.+Mounting the file per container avoids the question. `nano-init` checks whether+`/etc/resolv.conf` already names its resolver and, if so, does nothing — so the+container needs no `securityContext` at all: no capabilities, not privileged, no+AppArmor exception.++**Why not `dnsConfig`?** Because `dnsPolicy` and `dnsConfig` are PodSpec fields+and apply to every container in the pod. Measured on GKE, the per-container+mount gives you this:++```console+[sandbox] nameserver 169.254.1.1+[node] search default.svc.cluster.local ... nameserver 34.118.224.10+[node] cluster DNS works+```++With `dnsConfig`, `sam-node` would get `169.254.1.1` as well and could no longer+resolve the control plane.++**If you do not supply one, it needs somewhere writable.** Without the mount+above, `nano-init` falls back to bind-mounting a private `resolv.conf` itself,+which needs a real file to bind and therefore a writable path — an `emptyDir` on+`/tmp` will do. That path also needs a mount namespace, and the bind is what+AppArmor refuses, so on a default GKE node the fallback fails and the supplied+file is the way through. Verified on GKE 1.35: with the file mounted, the+sandbox starts under `cri-containerd.apparmor.d` with no `securityContext`;+without it, `nano-init` reports `permission denied` creating the namespaces.### The harness must be nano-init's childdiff --git a/tests/e2e/canary_manifests.bats b/tests/e2e/canary_manifests.batsindex 9e018d29..c7cb43f5 100644--- a/tests/e2e/canary_manifests.bats+++ b/tests/e2e/canary_manifests.bats@@ -102,6 +102,26 @@ unservable() {[[ "${failed}" -eq 0 ]]}+@test "Canary manifests: the sandbox resolv.conf names nano-init's own resolver" {+ # The canary hands the sandbox a resolv.conf so nano-init does not have to+ # mount one, and that only works while the address in it is the address+ # nano-init answers on. A mismatch is quiet in the wrong way: nano-init falls+ # back to the bind mount, which kind permits and containerd's default AppArmor+ # profile denies, so it would pass here and fail on the cluster.+ local declared resolver+ declared=$(grep -A2 'resolv.conf: |' .github/k8s/sam-box-canary-template.yaml \+ | grep nameserver | awk '{print $2}')+ resolver=$(grep -oE 'tunIP[[:space:]]*=[[:space:]]*"[0-9.]+"' cmd/nano-init/main.go \+ | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+')++ echo "# canary declares: ${declared}" >&3+ echo "# nano-init serves: ${resolver}" >&3++ [[ -n "${declared}" ]]+ [[ -n "${resolver}" ]]+ [[ "${declared}" == "${resolver}" ]]+}+@test "Canary rollout: the sandbox canary comes up and is a sandbox" {# deploy.yaml pulls these from ghcr; here they are the images this commit# just built, retagged so the manifest under test is the manifest shipped