Skip to content

Agent Mesh

canary: make the sandbox canary a sandbox, and check it in CI

Antonio Ojea1ec1edb657cc
diff --git a/.github/k8s/sam-box-canary-template.yaml b/.github/k8s/sam-box-canary-template.yaml
index e0dc0be4..31930c66 100644
--- a/.github/k8s/sam-box-canary-template.yaml
+++ b/.github/k8s/sam-box-canary-template.yaml
@@ -22,8 +22,16 @@ metadata:
# allowlisted destination through the boundary, and nothing else.
#
# The node enrols and owns the mesh identity; sam-box consumes it over the API
-# socket and serves the sandbox; the agent speaks plain SOCKS5. All that is
-# shared between them is the socket, which is the point of the arrangement.
+# socket and serves the sandbox. All that is shared between them is the socket,
+# which is the point of the arrangement.
+#
+# The agent is a real sandbox rather than a client that has been told where to
+# find a proxy. Every container in a pod shares one network namespace, so
+# nano-init makes its own with --create-namespaces and gives itself the only
+# route out; the agent then speaks ordinary HTTP to ordinary names, and the
+# fact that it reaches anything at all is the assertion. A curl pointed at
+# --socks5-hostname would prove the boundary works for a client that chose to
+# use it, which is the one case nobody needs proving.
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -40,6 +48,16 @@ spec:
app: box-canary-${ENV_NAME}
spec:
serviceAccountName: sam-box-sa
+ # nano-init ships as its own image with no shell in it, so the binary is
+ # handed to the agent container rather than the agent image being built
+ # around it. Any image with a client in it can be a sandbox this way.
+ initContainers:
+ - name: nano-init
+ image: ghcr.io/google/sam-nano-init:${IMAGE_TAG}
+ args: ["copy", "/sandbox/nano-init"]
+ volumeMounts:
+ - name: sandbox
+ mountPath: /sandbox
containers:
- name: sam-node
image: ghcr.io/google/sam-node:${IMAGE_TAG}
@@ -85,31 +103,34 @@ spec:
volumeMounts:
- name: sam-uds
mountPath: /var/run/sam
- # Stands in for the tun2socks a real sandbox runs: the boundary is reached
- # over a socket, and an ordinary client needs a TCP address.
- - name: socks-bridge
- image: alpine/socat:1.8.0.0
- args: ["TCP-LISTEN:1080,fork,reuseaddr", "UNIX-CONNECT:/var/run/sam/agent.sock"]
- resources:
- requests:
- cpu: 10m
- memory: 16Mi
- limits:
- cpu: 50m
- memory: 64Mi
- volumeMounts:
- - name: sam-uds
- mountPath: /var/run/sam
+ # No securityContext: creating the namespaces needs CAP_SYS_ADMIN and the
+ # tun needs CAP_NET_ADMIN, and a user namespace supplies both over the
+ # namespaces it owns. Granting CAP_SYS_ADMIN alone would be worse than
+ # granting nothing -- the namespace would be created and the tun would
+ # then fail for want of the other one.
- name: agent
image: alpine/curl:8.12.1
+ # The boundary socket is 0600, owned by the uid sam-box runs as, and
+ # that is the access control rather than an accident of packaging. A
+ # user namespace maps exactly one uid, so the sandbox has to be that
+ # uid: as root it would map 0 and hold no privilege over a file owned
+ # by 65532, and the connection would be refused.
+ securityContext:
+ runAsUser: 65532
+ runAsGroup: 65532
command:
+ - "/sandbox/nano-init"
+ - "run"
+ - "--create-namespaces"
+ - "/var/run/sam/agent.sock"
- "sh"
- "-c"
- |
+ echo "the sandbox's interfaces (expect lo and tun0 only): $(ip -o link show | cut -d: -f2 | tr -d ' ' | paste -sd,)"
while true; do
- echo "allowlisted destination (expect 200): $(curl -s -o /dev/null -w '%{http_code}' --socks5-hostname 127.0.0.1:1080 http://example.com/)"
- echo "the node's own API (expect 403): $(curl -s -o /dev/null -w '%{http_code}' --socks5-hostname 127.0.0.1:1080 http://mesh.sam.alt/sam/service/register)"
- curl -s -o /dev/null --socks5-hostname 127.0.0.1:1080 http://blocked.example/ && echo "unlisted destination was NOT refused" || echo "unlisted destination refused (expected)"
+ echo "allowlisted destination (expect 200): $(curl -s -o /dev/null -w '%{http_code}' http://example.com/)"
+ echo "the node's own API (expect 403): $(curl -s -o /dev/null -w '%{http_code}' http://mesh.sam.alt/sam/service/register)"
+ curl -s -o /dev/null http://blocked.example/ && echo "unlisted destination was NOT refused" || echo "unlisted destination refused (expected)"
sleep 30
done
resources:
@@ -119,9 +140,32 @@ spec:
limits:
cpu: 50m
memory: 64Mi
+ volumeMounts:
+ - name: sam-uds
+ mountPath: /var/run/sam
+ - name: sandbox
+ mountPath: /sandbox
+ # A new mount namespace copies the mount table, not the files behind
+ # it, so the private resolv.conf is a bind mount over a real file that
+ # has to be created somewhere.
+ - name: scratch
+ mountPath: /tmp
+ # A bind mount is enough. The device cgroup does not deny this one, so
+ # there is no device plugin involved; what the tun needs is the
+ # capability, and the user namespace supplies that.
+ - name: tun
+ mountPath: /dev/net/tun
volumes:
- name: sam-uds
emptyDir: {}
+ - name: sandbox
+ emptyDir: {}
+ - name: scratch
+ emptyDir: {}
+ - name: tun
+ hostPath:
+ path: /dev/net/tun
+ type: CharDevice
- name: config-volume
configMap:
name: sam-canary-box-config-${ENV_NAME}
diff --git a/tests/e2e/canary_manifests.bats b/tests/e2e/canary_manifests.bats
new file mode 100644
index 00000000..9e018d29
--- /dev/null
+++ b/tests/e2e/canary_manifests.bats
@@ -0,0 +1,175 @@
+#!/usr/bin/env bats
+#
+# The canary manifests in .github/k8s are applied straight to a live testnet by
+# .github/workflows/deploy.yaml, and until now the first thing to find out
+# whether they were valid was that testnet. A field renamed by a chart bump, an
+# apiVersion that lapsed, a template variable nobody substitutes -- each of
+# those is a broken prod rollout discovered in prod.
+#
+# This applies them to the e2e cluster first. Three things are asserted, and
+# they fail for different reasons: that the rendered manifest is one an API
+# server accepts, that the deployment it describes rolls out, and that the
+# sandbox inside it is a sandbox.
+#
+# The last one is worth having here rather than only in agent_sandbox.bats,
+# because the pod is the profile where nothing isolates the agent for you.
+# Every container in a pod shares one network namespace, so a canary that came
+# up Ready would say nothing at all about whether the agent was confined.
+
+load "lib/container_mesh.bash"
+
+# The values deploy.yaml substitutes. ENV_NAME picks the canary namespace and
+# the control plane's service name; NAMESPACE is where the mesh itself lives,
+# which in this cluster is the namespace helm installed into.
+export ENV_NAME="e2e"
+export NAMESPACE="default"
+export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"
+
+setup_file() {
+ if ! command -v kind >/dev/null 2>&1 || ! command -v kubectl >/dev/null 2>&1; then
+ skip "canary manifests need a cluster: kind and kubectl are not both available"
+ fi
+ if ! command -v envsubst >/dev/null 2>&1; then
+ skip "envsubst is how deploy.yaml renders these, so it is how they are rendered here"
+ fi
+}
+
+# Cleanup is per-file rather than per-test: deleting a namespace is
+# asynchronous, and a second test that recreates it races the first one's
+# termination.
+teardown_file() {
+ local kubectl="kubectl --context=${KUBECONTEXT:-kind-${KUBERNETES_CLUSTER_NAME:-sam-wi-test}}"
+ ${kubectl} delete namespace "${CANARY_NAMESPACE}" --ignore-not-found --wait=false >/dev/null 2>&1 || true
+ ${kubectl} -n "${NAMESPACE}" delete service "sam-control-plane-${ENV_NAME}" \
+ --ignore-not-found --wait=false >/dev/null 2>&1 || true
+}
+
+setup() {
+ mesh_setup_env
+ export KUBECTL="kubectl --context=${KUBECONTEXT:-kind-${KUBERNETES_CLUSTER_NAME:-sam-wi-test}}"
+ ${KUBECTL} create namespace "${CANARY_NAMESPACE}" --dry-run=client -o yaml | ${KUBECTL} apply -f - >/dev/null
+}
+
+teardown() {
+ mesh_cleanup_env
+}
+
+# render fills in the variables deploy.yaml substitutes.
+render() {
+ IMAGE_TAG="${IMAGE_TAG:-local}" envsubst < "$1"
+}
+
+# unservable reports whether every error is this cluster's missing CRDs rather
+# than the manifest's fault. The control plane template carries GKE Gateway and
+# HealthCheckPolicy resources, which no kind cluster serves and which nothing
+# here can install, so failing on them would only teach people to ignore this
+# test.
+#
+# The kinds are named rather than matched on "no matches for kind", because a
+# Deployment declared against a retired apiVersion fails with that same wording
+# -- which made an earlier version of this test pass a manifest whose
+# apps/v1beta1 would have broken the rollout it exists to protect.
+unservable() {
+ local kinds='Gateway|HTTPRoute|HealthCheckPolicy|GCPBackendPolicy'
+ ! grep -qvE "no matches for kind \"(${kinds})\"|ensure CRDs are installed|^[[:space:]]*$" <<<"$1"
+}
+
+@test "Canary manifests: every template the deploy workflow applies is valid" {
+ local template errs failed=0
+
+ for template in .github/k8s/*-template.yaml; do
+ # A CronJob against a DNS zone this cluster has no notion of.
+ case "${template}" in
+ *dns-sync*) continue ;;
+ esac
+
+ # Server-side, so this is the API server's own schema rather than a local
+ # guess: it catches the field a chart bump renamed, which client-side
+ # validation accepts happily.
+ if errs=$(render "${template}" | ${KUBECTL} apply --dry-run=server -f - 2>&1 >/dev/null); then
+ echo "# ok ${template}" >&3
+ continue
+ fi
+ if unservable "${errs}"; then
+ echo "# skipped ${template} (needs CRDs this cluster does not serve)" >&3
+ continue
+ fi
+ echo "# INVALID ${template}" >&3
+ sed 's/^/# /' <<<"${errs}" >&3
+ failed=1
+ done
+
+ [[ "${failed}" -eq 0 ]]
+}
+
+@test "Canary rollout: the sandbox canary comes up and is a sandbox" {
+ # deploy.yaml pulls these from ghcr; here they are the images this commit
+ # just built, retagged so the manifest under test is the manifest shipped
+ # rather than a copy with the registry edited out.
+ local image
+ for image in sam-node sam-box sam-nano-init; do
+ docker image inspect "${image}:local" >/dev/null 2>&1 \
+ || skip "${image}:local not built; run make docker-build"
+ docker tag "${image}:local" "ghcr.io/google/${image}:local"
+ kind load docker-image "ghcr.io/google/${image}:local" \
+ --name "${KUBERNETES_CLUSTER_NAME:-sam-wi-test}" >/dev/null
+ done
+
+ # The canary addresses the control plane by the name deploy.yaml gives it.
+ # An alias keeps the manifest unedited, which is the point of the exercise.
+ ${KUBECTL} apply -f - <<EOF
+apiVersion: v1
+kind: Service
+metadata:
+ name: sam-control-plane-${ENV_NAME}
+ namespace: ${NAMESPACE}
+spec:
+ type: ExternalName
+ externalName: sam-control-plane.${NAMESPACE}.svc.cluster.local
+EOF
+
+ render .github/k8s/sam-box-canary-template.yaml | ${KUBECTL} apply -f -
+
+ run ${KUBECTL} -n "${CANARY_NAMESPACE}" rollout status \
+ "deployment/box-canary-${ENV_NAME}" --timeout=120s
+ if [[ "${status}" -ne 0 ]]; then
+ echo "# rollout did not complete:" >&3
+ ${KUBECTL} -n "${CANARY_NAMESPACE}" get pods -o wide 2>&1 | sed 's/^/# /' >&3
+ ${KUBECTL} -n "${CANARY_NAMESPACE}" describe "deployment/box-canary-${ENV_NAME}" 2>&1 \
+ | sed 's/^/# /' >&3
+ ${KUBECTL} -n "${CANARY_NAMESPACE}" logs "deployment/box-canary-${ENV_NAME}" \
+ --all-containers --tail=40 2>&1 | sed 's/^/# /' >&3
+ fi
+ [[ "${status}" -eq 0 ]]
+
+ # Every container the manifest declares has to be up, not merely enough of
+ # them for the pod to report Ready.
+ run ${KUBECTL} -n "${CANARY_NAMESPACE}" get pods \
+ -l "app=box-canary-${ENV_NAME}" \
+ -o jsonpath='{.items[0].status.containerStatuses[*].ready}'
+ [[ "${status}" -eq 0 ]]
+ [[ -n "${output}" ]]
+ [[ "${output}" != *"false"* ]]
+
+ # The agent loops every 30s; the first pass is what is being waited for.
+ local logs="" i
+ for ((i = 0; i < 60; i++)); do
+ logs=$(${KUBECTL} -n "${CANARY_NAMESPACE}" logs "deployment/box-canary-${ENV_NAME}" \
+ -c agent --tail=40 2>/dev/null || true)
+ [[ "${logs}" == *"unlisted destination"* ]] && break
+ sleep 2
+ done
+ echo "# agent reported:" >&3
+ sed 's/^/# /' <<<"${logs}" >&3
+
+ # The pod has an eth0 and a route to the cluster. The agent must have
+ # neither, which is the one thing a Ready pod would not have told us.
+ [[ "${logs}" == *"interfaces (expect lo and tun0 only): lo,tun0"* ]]
+
+ # And it still reaches what policy allows, is refused the node's own admin
+ # surface, and cannot reach a name nobody allowed.
+ [[ "${logs}" == *"allowlisted destination (expect 200): 200"* ]]
+ [[ "${logs}" == *"the node's own API (expect 403): 403"* ]]
+ [[ "${logs}" == *"unlisted destination refused (expected)"* ]]
+ [[ "${logs}" != *"was NOT refused"* ]]
+}