Skip to content

Agent Mesh

ci: let the sandbox canary keep the capabilities its namespace grants

Antonio Ojeafde9b832b34b
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 9891407d..8280be03 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -51,6 +51,20 @@ jobs:
id: setup-bats
uses: bats-core/bats-action@77d6fb60505b4d0d1d73e48bd035b55074bbfb43 # 4.0.0
+ # The sandbox canary builds its own namespaces from inside a pod, and a
+ # user namespace is what grants it CAP_NET_ADMIN for the tun. Ubuntu
+ # 24.04 confines an AppArmor-unconfined process that creates one to the
+ # unprivileged_userns profile, which denies every capability -- so the
+ # namespace is created and the tun is then refused. kind runs its pods
+ # unconfined, so every sandbox in this cluster is that process.
+ - name: Let an unprivileged user namespace keep its capabilities
+ shell: bash
+ run: |
+ restrict=/proc/sys/kernel/apparmor_restrict_unprivileged_userns
+ if [[ -e "${restrict}" ]]; then
+ sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+ fi
+
- name: Bats tests
shell: bash
env:
diff --git a/cmd/nano-init/isolation.go b/cmd/nano-init/isolation.go
index 81da3c5a..930593be 100644
--- a/cmd/nano-init/isolation.go
+++ b/cmd/nano-init/isolation.go
@@ -80,10 +80,26 @@ func isolationError(links []string) error {
// is the whole reason for the diagnosis below.
const tunDevice = "/dev/net/tun"
+// userNSRestriction is Ubuntu's switch for what an unprivileged user namespace
+// may do.
+const userNSRestriction = "/proc/sys/kernel/apparmor_restrict_unprivileged_userns"
+
+// userNSCapabilitiesRestricted reports whether this host takes back the
+// capabilities a user namespace would otherwise grant.
+//
+// Where it is on, a process with no AppArmor profile that creates a user
+// namespace is confined to the unprivileged_userns profile, and that profile
+// denies every capability. Nothing fails at the time: the namespace is created,
+// and the capability inside it is refused later.
+func userNSCapabilitiesRestricted() bool {
+ value, err := os.ReadFile(userNSRestriction)
+ return err == nil && strings.TrimSpace(string(value)) != "0"
+}
+
// describeTunFailure turns a netlink error into the thing to change.
func describeTunFailure(err error) string {
_, statErr := os.Stat(tunDevice)
- return tunHint(err, statErr)
+ return tunHint(err, statErr, userNSCapabilitiesRestricted())
}
// tunHint explains a failed tun creation.
@@ -93,7 +109,7 @@ func describeTunFailure(err error) string {
// means the device was not passed in or the capability was not granted. The
// profiles fail differently, so they are told apart here rather than left to
// whoever is reading a log at the time.
-func tunHint(err error, statErr error) string {
+func tunHint(err error, statErr error, restrictedUserNS bool) string {
switch {
case os.IsNotExist(statErr):
return "there is no " + tunDevice + ". In a microVM that means a guest kernel built without CONFIG_TUN" +
@@ -110,6 +126,16 @@ func tunHint(err error, statErr error) string {
// netlink formats this one rather than wrapping the errno, so there is
// nothing for errors.Is to match on.
err != nil && strings.Contains(err.Error(), "TUNSETIFF"):
+ // Named first because the advice below is wrong here: the capability
+ // was granted and then taken away again, so granting it harder is no
+ // answer.
+ if restrictedUserNS {
+ return "creating a tun was refused, and this host restricts what an unprivileged user namespace may do" +
+ " (" + userNSRestriction + " is not 0): a process with no AppArmor profile that creates one is" +
+ " confined to the unprivileged_userns profile, which denies every capability -- so the namespace" +
+ " was created and CAP_NET_ADMIN in it was refused anyway. Give the sandbox an AppArmor profile" +
+ " that permits capabilities, or set that sysctl to 0"
+ }
return "creating a tun was refused. It needs CAP_NET_ADMIN in the user namespace that owns this network" +
" namespace: a user namespace of your own grants it over the namespaces it owns, which is what" +
" --create-namespaces relies on; otherwise `--cap-add NET_ADMIN` for docker, or" +
diff --git a/cmd/nano-init/isolation_test.go b/cmd/nano-init/isolation_test.go
index 38257b75..2223e363 100644
--- a/cmd/nano-init/isolation_test.go
+++ b/cmd/nano-init/isolation_test.go
@@ -30,6 +30,9 @@ func TestTunHint(t *testing.T) {
name string
err error
statErr error
+ // restricted is a host that hands back a user namespace and then
+ // denies the capabilities it would have granted.
+ restricted bool
// want is a phrase naming the fix, so the test fails when the advice
// stops matching the cause rather than only when the wording changes.
want string
@@ -63,6 +66,21 @@ func TestTunHint(t *testing.T) {
err: syscall.EPERM,
want: "CAP_NET_ADMIN",
},
+ {
+ // The capability advice is worse than useless here: it was granted
+ // by the namespace and taken back by the host.
+ name: "a host that strips a user namespace's capabilities names the sysctl",
+ err: syscall.EPERM,
+ restricted: true,
+ want: "apparmor_restrict_unprivileged_userns",
+ },
+ {
+ name: "a device that cannot be opened is still not blamed on the sysctl",
+ err: syscall.EPERM,
+ statErr: denied,
+ restricted: true,
+ want: "cannot be opened",
+ },
{
name: "EACCES is treated as EPERM is",
err: syscall.EACCES,
@@ -80,9 +98,9 @@ func TestTunHint(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
- got := tunHint(tc.err, tc.statErr)
+ got := tunHint(tc.err, tc.statErr, tc.restricted)
if !strings.Contains(got, tc.want) {
- t.Errorf("tunHint(%v, %v) = %q, want it to mention %q", tc.err, tc.statErr, got, tc.want)
+ t.Errorf("tunHint(%v, %v, %v) = %q, want it to mention %q", tc.err, tc.statErr, tc.restricted, got, tc.want)
}
})
}