Skip to content

Agent Mesh

sandbox: give the pod profile the AppArmor it actually needs

Antonio Ojea60ae03d7a7bb
diff --git a/.github/k8s/sam-box-canary-template.yaml b/.github/k8s/sam-box-canary-template.yaml
index 31930c66..7d113382 100644
--- a/.github/k8s/sam-box-canary-template.yaml
+++ b/.github/k8s/sam-box-canary-template.yaml
@@ -115,9 +115,18 @@ spec:
# user namespace maps exactly one uid, so the sandbox has to be that
# uid: as root it would map 0 and hold no privilege over a file owned
# by 65532, and the connection would be refused.
+ #
+ # Unconfined AppArmor is the one relaxation the sandbox needs. The
+ # profile containerd applies by default permits the mount namespace and
+ # then denies the bind mount inside it, so the sandbox would get a
+ # namespace it cannot give a private resolv.conf. Everything else stays
+ # off: no capabilities, no privileged, and a user namespace that holds
+ # the process to what it owns.
securityContext:
runAsUser: 65532
runAsGroup: 65532
+ appArmorProfile:
+ type: Unconfined
command:
- "/sandbox/nano-init"
- "run"
diff --git a/cmd/nano-init/namespaces.go b/cmd/nano-init/namespaces.go
index 9e1361f9..3c59c0cd 100644
--- a/cmd/nano-init/namespaces.go
+++ b/cmd/nano-init/namespaces.go
@@ -172,8 +172,23 @@ func userNamespacesAvailable() error {
return nil
}
-// privateResolvConf gives the sandbox a resolv.conf of its own.
+// mountHint explains a refused mount, which in a container is usually a
+// sandboxing profile rather than a missing capability.
//
+// Kubernetes is the case worth naming. containerd applies an AppArmor profile
+// by default -- cri-containerd.apparmor.d on GKE -- and it denies bind mounts
+// while allowing the namespace itself, so the sandbox gets as far as having a
+// mount namespace and then cannot put anything in it.
+func mountHint(err error) string {
+ if !errors.Is(err, syscall.EPERM) && !errors.Is(err, syscall.EACCES) {
+ return ""
+ }
+ return " This is usually AppArmor rather than a missing capability: containerd applies a" +
+ " profile that denies bind mounts. In a pod, set the agent container's" +
+ " securityContext.appArmorProfile.type to Unconfined, or load a profile that permits" +
+ " mount. Docker's equivalent is --security-opt apparmor=unconfined."
+}
+
// A new mount namespace copies the mount table; it does not copy files. The
// resolv.conf a pod hands each container is one file shared by all of them, so
// without this, pointing DNS at the sandbox's own resolver would point every
@@ -194,7 +209,7 @@ func privateResolvConf() error {
}
if err := unix.Mount(name, "/etc/resolv.conf", "", unix.MS_BIND, ""); err != nil {
_ = os.Remove(name)
- return fmt.Errorf("bind a private resolv.conf over /etc/resolv.conf: %w", err)
+ return fmt.Errorf("bind a private resolv.conf over /etc/resolv.conf: %w.%s", err, mountHint(err))
}
// The mount holds the inode, so the path it was reached by is litter.
_ = os.Remove(name)
diff --git a/site/content/docs/user/running-agents.md b/site/content/docs/user/running-agents.md
index 0953cc27..900b18a4 100644
--- a/site/content/docs/user/running-agents.md
+++ b/site/content/docs/user/running-agents.md
@@ -340,6 +340,10 @@ spec:
# starts, no network.
- name: agent
image: your-agent-image
+ securityContext:
+ # The one relaxation the sandbox needs; see below.
+ appArmorProfile:
+ type: Unconfined
command: ["/usr/local/bin/nano-init", "run", "--create-namespaces",
"/var/run/sam/agent.sock", "python3", "/app/agent.py"]
volumeMounts:
@@ -365,14 +369,35 @@ holds both over the namespaces it then makes. Granting `CAP_SYS_ADMIN` alone is
in fact worse than granting nothing: the namespace gets created, and then the
tun fails for want of `CAP_NET_ADMIN`.
-**A seccomp and AppArmor policy that permits it.** This is the one that bites.
-Creating a user namespace and remounting `/` as private are exactly the
-operations container sandboxing profiles restrict. Kubernetes applies neither
-profile unless you ask it to, so a default pod works — but a cluster with
-`seccompDefault=RuntimeDefault`, or a `runtime/default` AppArmor profile, must
-permit `unshare(CLONE_NEWUSER|CLONE_NEWNS)` and `mount`. Docker's defaults block
-both, which is why the same image needs `--security-opt seccomp=unconfined
---security-opt apparmor=unconfined` to be tried locally.
+**A seccomp and AppArmor policy that permits it.** This is the one that bites,
+and it is the reason for the `appArmorProfile` above. containerd applies an
+AppArmor profile by default — `cri-containerd.apparmor.d` on GKE — which permits
+creating the mount namespace and then **denies the bind mount inside it**. The
+sandbox gets a namespace it cannot put a private `resolv.conf` into. Measured on
+a GKE 1.35 node:
+
+```console
+user+net : ok
+user+mount (unchanged propagation): ok
+bind mount inside : FAILED
+```
+
+With `appArmorProfile: {type: Unconfined}` on that container, all three pass.
+Seccomp is not involved: GKE reports `Seccomp: 0` for pods that do not ask for a
+profile, and creating the namespaces is allowed. Docker is the other way round —
+its default seccomp blocks the user namespace and its default AppArmor blocks the
+mount — which is why running the same image locally needs
+`--security-opt seccomp=unconfined --security-opt apparmor=unconfined`.
+
+It is worth being clear about what that costs, because the container it applies
+to is the least trusted one. The agent keeps every other constraint: no
+capabilities, not privileged, its own user namespace, and no network but the
+tun. What it gains is the ability to mount inside its own mount namespace, which
+a user namespace already confines to filesystems it owns. The tighter option is
+a custom profile — the default plus `mount` — loaded on the nodes and selected
+with `appArmorProfile: {type: Localhost, localhostProfile: ...}`; that is
+strictly better and costs an operational step, so it is the right thing to move
+to rather than the right thing to start with.
**Somewhere writable.** A new mount namespace copies the mount table, not the
files behind it, so a private `resolv.conf` is a bind mount over a real file,
diff --git a/tests/integration/sandbox_ingress_test.go b/tests/integration/sandbox_ingress_test.go
index 8886df9a..87530403 100644
--- a/tests/integration/sandbox_ingress_test.go
+++ b/tests/integration/sandbox_ingress_test.go
@@ -130,9 +130,15 @@ func TestSandboxServesThroughTheReverseChannel(t *testing.T) {
var resp *http.Response
for time.Now().Before(deadline) {
- req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet,
+ // Each attempt is bounded on its own. Sharing the outer deadline meant
+ // one attempt that hung -- against a listener the sandbox had not
+ // restarted yet -- spent the whole budget and failed the test under
+ // load, while passing whenever it was run alone.
+ attemptCtx, cancelAttempt := context.WithTimeout(ctx, 5*time.Second)
+ req, reqErr := http.NewRequestWithContext(attemptCtx, http.MethodGet,
fmt.Sprintf("http://127.0.0.1:%d/", agentPort), nil)
if reqErr != nil {
+ cancelAttempt()
t.Fatalf("build the request: %v", reqErr)
}
// The agent inside serves one connection at a time, so reusing one
@@ -140,8 +146,10 @@ func TestSandboxServesThroughTheReverseChannel(t *testing.T) {
req.Close = true
resp, err = client.Do(req)
if err == nil {
+ defer cancelAttempt()
break
}
+ cancelAttempt()
time.Sleep(200 * time.Millisecond)
}
if err != nil {