Skip to content

Agent Mesh

docs, examples: OpenShell and Substrate on one mesh, on your Kubernetes or on GKE

Antonio Ojea1851f146247f
diff --git a/development/examples/openshell-kubernetes/.gitignore b/development/examples/openshell-kubernetes/.gitignore
new file mode 100644
index 00000000..7e3b24e0
--- /dev/null
+++ b/development/examples/openshell-kubernetes/.gitignore
@@ -0,0 +1 @@
+run/
diff --git a/development/examples/openshell-kubernetes/Dockerfile.sandbox b/development/examples/openshell-kubernetes/Dockerfile.sandbox
new file mode 100644
index 00000000..ea66c9dd
--- /dev/null
+++ b/development/examples/openshell-kubernetes/Dockerfile.sandbox
@@ -0,0 +1,17 @@
+# The sandbox image: Ubuntu with curl and python3 for the walkthrough, built
+# the way OpenShell's bring-your-own-container example does it. An agent image
+# (Claude Code, Codex, Gemini CLI, ...) takes the same place.
+FROM nvcr.io/nvidia/base/ubuntu:24.04
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+ bash ca-certificates curl iproute2 nftables python3 \
+ && rm -rf /var/lib/apt/lists/* \
+ && if getent passwd ubuntu >/dev/null; then userdel ubuntu; fi \
+ && if getent group ubuntu >/dev/null; then groupdel ubuntu; fi \
+ && groupadd --gid 1000 sandbox \
+ && useradd --uid 1000 --gid sandbox --create-home --shell /bin/bash sandbox \
+ && install -d -o sandbox -g sandbox /sandbox
+
+WORKDIR /sandbox
+USER sandbox:sandbox
diff --git a/development/examples/openshell-kubernetes/Makefile b/development/examples/openshell-kubernetes/Makefile
new file mode 100644
index 00000000..64de7ef1
--- /dev/null
+++ b/development/examples/openshell-kubernetes/Makefile
@@ -0,0 +1,281 @@
+# An agent in an OpenShell sandbox on Kubernetes, on the mesh under its own
+# identity. Nothing in OpenShell is modified: the Helm chart is installed as
+# published with one value set, and the integration is one provider profile.
+#
+# make cluster kind cluster (skip on a cluster you already have)
+# make prereqs Agent Sandbox controller, SPIRE (server, agent, CSI, OIDC)
+# make mesh the mesh: hub (HUB=local) or join HUB=<url>, edge + office nodes
+# make openshell OpenShell gateway via Helm, CLI registration, the provider
+# make up prereqs + mesh + openshell
+# make sandbox NAME=demo create a sandbox with the provider; prints its SPIFFE ID
+# make bind NAME=demo add that sandbox's identity to the sandbox role and push
+# make unbind NAME=demo remove it again (revocation is a policy change)
+# make call NAME=demo from inside the sandbox: tools, calc allowed, payroll refused
+# make rogue a sandbox the policy does not bind: refused
+# make netpol-check prove the CNI enforces NetworkPolicy (workload cannot bypass the supervisor)
+# make policy push policy.json to the hub
+# make down remove sandboxes, OpenShell, the mesh, SPIRE
+#
+# Needs: kubectl, helm, docker (to build the sandbox image), the openshell CLI
+# (0.1.4 or later), python3, curl. KUBE_CONTEXT selects the cluster. For kind,
+# `make cluster` creates it and loads images; on GKE set
+# OIDC_SERVICE_TYPE=LoadBalancer so the hub on Cloud Run can reach SPIRE's
+# issuer, and HUB to the Cloud Run URL.
+
+SHELL := /bin/bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+
+NAME ?= demo
+# Local port for the port-forward to the OpenShell gateway.
+GATEWAY_PORT ?= 18080
+KIND_CLUSTER_NAME ?= openshell
+KUBE_CONTEXT ?= kind-$(KIND_CLUSTER_NAME)
+K := kubectl --context $(KUBE_CONTEXT)
+H := helm --kube-context $(KUBE_CONTEXT)
+CLUSTER_NAME ?= $(KIND_CLUSTER_NAME)
+TRUST_DOMAIN := openshell.local
+
+AGENT_SANDBOX_VERSION ?= v1.0.7
+SPIRE_CRDS_VERSION ?= 0.5.0
+SPIRE_CHART_VERSION ?= 0.29.0
+OPENSHELL_VERSION ?= 0.1.4
+# Name of this cluster's gateway in the openshell CLI; use one name per cluster.
+OPENSHELL_GATEWAY ?= openshell-k8s
+AGENTMESH_VERSION ?= v0.1.1
+AGENTMESH_ONE_IMAGE ?= ghcr.io/google/agentmesh-one:$(AGENTMESH_VERSION)
+AGENTMESH_NODE_IMAGE ?= ghcr.io/google/agentmesh-node:$(AGENTMESH_VERSION)
+CALC_MCP_IMAGE ?= calc-mcp:local
+SANDBOX_IMAGE ?= openshell-agentmesh-sandbox:local
+
+# HUB=local runs agentmesh-one in the cluster; HUB=https://... joins one that
+# already runs elsewhere (Cloud Run). The admin token for a remote hub comes
+# from run/admin-token (write it there); the join token from run/join-token.
+HUB ?= local
+ifeq ($(HUB),local)
+CONTROL_PLANE_URL := http://agentmesh-one.agentmesh.svc.cluster.local:8080
+CONTROL_PLANE_INSECURE_ARG := - --insecure-control-plane
+HUB_URL_LOCAL := http://127.0.0.1:18070
+else
+CONTROL_PLANE_URL := $(HUB)
+CONTROL_PLANE_INSECURE_ARG :=
+HUB_URL_LOCAL := $(HUB)
+endif
+
+# The issuer SPIRE's OIDC discovery provider names in every JWT-SVID and the
+# control plane fetches keys from. In-cluster by default; on GKE a LoadBalancer
+# Service and JWT_ISSUER=http://<ip> so the hub on Cloud Run can reach it.
+OIDC_SERVICE_TYPE ?= ClusterIP
+OIDC_LOAD_BALANCER_IP ?=
+JWT_ISSUER ?= http://spire-spiffe-oidc-discovery-provider.spire.svc.cluster.local
+
+export KIND_CLUSTER_NAME KUBE_CONTEXT
+
+.PHONY: cluster prereqs mesh openshell up sandbox bind unbind call rogue gemini netpol-check policy status down clean
+
+cluster:
+ kind get clusters 2>/dev/null | grep -qx "$(KIND_CLUSTER_NAME)" || kind create cluster --name $(KIND_CLUSTER_NAME) --wait 120s
+
+prereqs:
+ $(K) apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/download/$(AGENT_SANDBOX_VERSION)/sandbox.yaml
+ $(K) -n agent-sandbox-system rollout status deployment --timeout=120s
+ helm repo add spire https://spiffe.github.io/helm-charts-hardened/ >/dev/null 2>&1 || true
+ helm repo update spire >/dev/null
+ $(H) upgrade --install spire-crds spire/spire-crds --version $(SPIRE_CRDS_VERSION) -n spire --create-namespace --wait
+ $(K) get namespace openshell >/dev/null 2>&1 || $(K) create namespace openshell
+ mkdir -p run
+ CLUSTER_NAME="$(CLUSTER_NAME)" JWT_ISSUER="$(JWT_ISSUER)" OIDC_SERVICE_TYPE="$(OIDC_SERVICE_TYPE)" OIDC_LOAD_BALANCER_IP="$(OIDC_LOAD_BALANCER_IP)" \
+ envsubst '$$CLUSTER_NAME $$JWT_ISSUER $$OIDC_SERVICE_TYPE $$OIDC_LOAD_BALANCER_IP' < spire/values.yaml.tmpl > run/spire-values.yaml
+ $(H) upgrade --install spire spire/spire --version $(SPIRE_CHART_VERSION) -n spire -f run/spire-values.yaml --wait --timeout 5m
+ @echo "SPIRE issuer: $(JWT_ISSUER)"
+
+mesh: run/join-token run/admin-token run/api-token
+ $(K) get namespace agentmesh >/dev/null 2>&1 || $(K) create namespace agentmesh
+ $(K) -n agentmesh get secret agentmesh-tokens >/dev/null 2>&1 || \
+ $(K) -n agentmesh create secret generic agentmesh-tokens \
+ --from-file=join-token=run/join-token --from-file=admin-token=run/admin-token --from-file=api-token=run/api-token
+ if [ "$(HUB)" = local ]; then
+ POLICY_JSON_INDENTED="$$(sed 's/^/ /' policy.json)" AGENTMESH_ONE_IMAGE="$(AGENTMESH_ONE_IMAGE)" JWT_ISSUER="$(JWT_ISSUER)" \
+ envsubst '$$POLICY_JSON_INDENTED $$AGENTMESH_ONE_IMAGE $$JWT_ISSUER' < mesh/hub.yaml.tmpl | $(K) apply -f -
+ fi
+ calc="$(CALC_MCP_IMAGE)"
+ if [ "$$calc" = calc-mcp:local ]; then
+ docker build -q -t calc-mcp:local ../calc-mcp >/dev/null
+ kind load docker-image calc-mcp:local --name $(KIND_CLUSTER_NAME) >/dev/null
+ calc=docker.io/library/calc-mcp:local
+ fi
+ AGENTMESH_NODE_IMAGE="$(AGENTMESH_NODE_IMAGE)" CALC_MCP_IMAGE="$$calc" CONTROL_PLANE_URL="$(CONTROL_PLANE_URL)" CONTROL_PLANE_INSECURE_ARG="$(CONTROL_PLANE_INSECURE_ARG)" \
+ envsubst '$$AGENTMESH_NODE_IMAGE $$CALC_MCP_IMAGE $$CONTROL_PLANE_URL $$CONTROL_PLANE_INSECURE_ARG' < mesh/nodes.yaml.tmpl | $(K) apply -f -
+ $(K) -n agentmesh rollout status deployment --timeout=180s
+ @echo "mesh is up; edge node for the sandboxes: http://edge-node.agentmesh.svc.cluster.local:8080"
+
+openshell:
+ @command -v openshell >/dev/null || { echo "openshell CLI not found: https://docs.openshell.ai/about/installation"; exit 1; }
+ $(H) upgrade --install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version $(OPENSHELL_VERSION) \
+ -n openshell --create-namespace -f openshell/values.yaml --wait --timeout 5m
+ # The chart's client mTLS bundle is the CLI's transport credential over a
+ # port-forward; the server certificate names 127.0.0.1.
+ mkdir -p run/mtls
+ for f in ca.crt tls.crt tls.key; do
+ $(K) -n openshell get secret openshell-client-tls -o jsonpath="{.data.$${f//./\\.}}" | base64 -d > run/mtls/$$f
+ done
+ mkdir -p ~/.config/openshell/gateways/$(OPENSHELL_GATEWAY)/mtls && cp run/mtls/* ~/.config/openshell/gateways/$(OPENSHELL_GATEWAY)/mtls/
+ EDGE_NODE_IP=$$($(K) -n agentmesh get svc edge-node -o jsonpath='{.spec.clusterIP}') \
+ envsubst '$$EDGE_NODE_IP' < openshell/provider-profile.yaml.tmpl > run/provider-profile.yaml
+ $(MAKE) --no-print-directory _with-gateway-port-forward CMD='\
+ openshell gateway list 2>/dev/null | grep -qw "$(OPENSHELL_GATEWAY)" || openshell gateway add https://127.0.0.1:$(GATEWAY_PORT) --local --name $(OPENSHELL_GATEWAY); \
+ openshell gateway select $(OPENSHELL_GATEWAY) >/dev/null; \
+ ./profile-apply.sh run/provider-profile.yaml; \
+ openshell provider list 2>/dev/null | grep -qw "agentmesh" || openshell provider create --name agentmesh --type agentmesh --runtime-credentials'
+ if [ "$(SANDBOX_IMAGE)" = openshell-agentmesh-sandbox:local ]; then
+ docker build -q -t $(SANDBOX_IMAGE) -f Dockerfile.sandbox . >/dev/null
+ kind load docker-image $(SANDBOX_IMAGE) --name $(KIND_CLUSTER_NAME) >/dev/null
+ fi
+ @echo "OpenShell $(OPENSHELL_VERSION) is up as gateway '$(OPENSHELL_GATEWAY)'; next: make sandbox NAME=demo"
+
+up: prereqs mesh openshell
+
+# The sandbox's identity on Kubernetes is derived from the sandbox id OpenShell
+# puts on the supervisor pod, so it is known only after creation. `bind` adds
+# it to the mesh policy.
+sandbox:
+ $(MAKE) --no-print-directory _with-gateway-port-forward CMD='\
+ openshell sandbox list 2>/dev/null | grep -qw "$(NAME)" || \
+ openshell sandbox create --name $(NAME) --from $(SANDBOX_IMAGE) --provider agentmesh --detach -- sleep infinity'
+ id=$$($(MAKE) --no-print-directory _with-gateway-port-forward CMD='openshell sandbox get $(NAME)' | awk '/^ *Id:/ {print $$2}' | head -1)
+ test -n "$$id" || { echo "could not read the id of sandbox $(NAME)"; exit 1; }
+ echo "$$id" > run/sandbox-id.$(NAME)
+ @echo "sandbox $(NAME) is spiffe://$(TRUST_DOMAIN)/openshell/sandbox/$$id; next: make bind NAME=$(NAME) && make call NAME=$(NAME)"
+
+bind:
+ python3 bind.py "user:spiffe://$(TRUST_DOMAIN)/openshell/sandbox/$$(cat run/sandbox-id.$(NAME))"
+ $(MAKE) --no-print-directory policy
+
+unbind:
+ python3 bind.py --remove "user:spiffe://$(TRUST_DOMAIN)/openshell/sandbox/$$(cat run/sandbox-id.$(NAME))"
+ $(MAKE) --no-print-directory policy
+
+call:
+ $(MAKE) --no-print-directory _with-gateway-port-forward CMD='\
+ echo "== tools the mesh offers sandbox $(NAME)"; python3 mcp.py tools $(NAME); \
+ echo "== mcp://calc, allowed for the sandbox role"; python3 mcp.py add $(NAME) calc 2 3; \
+ echo "== mcp://payroll, not in the sandbox role"; python3 mcp.py add $(NAME) payroll 2 3 || true'
+
+# gemini calls Gemini on Vertex AI from inside the sandbox through the mesh:
+# the request goes to the edge node, which forwards it to the office node that
+# serves egress://aiplatform.googleapis.com; that node authorizes the method
+# and path, mints a federated Google token and injects it. The sandbox holds
+# no Google credential. The policy must carry the egress destination and the
+# http grant (see the guide); GCP_PROJECT is the project of the destination.
+GCP_PROJECT ?=
+GCP_REGION ?= us-central1
+GEMINI_MODEL ?= gemini-2.5-flash
+# AGENT_ENGINE=<reasoning engine id> adds a query to an agent deployed on
+# Agent Engine, the same host under another granted path.
+AGENT_ENGINE ?=
+gemini:
+ @test -n "$(GCP_PROJECT)" || { echo "set GCP_PROJECT=<project that hosts the Vertex AI endpoint>"; exit 1; }
+ peer="$(PEER)"
+ if [ -z "$$peer" ]; then
+ peer=$$($(K) -n agentmesh logs deploy/office-node | grep -o 'PeerID: 12D3Koo[A-Za-z0-9]*' | head -1 | cut -d' ' -f2)
+ fi
+ base="http://edge-node.agentmesh.svc.cluster.local:8080/mesh/$$peer/egress/aiplatform.googleapis.com/v1/projects/$(GCP_PROJECT)/locations/$(GCP_REGION)"
+ $(MAKE) --no-print-directory _with-gateway-port-forward CMD="\
+ echo '== POST $(GEMINI_MODEL):generateContent, granted to the sandbox role'; \
+ openshell sandbox exec --name $(NAME) --no-tty -- curl -sS --max-time 60 -H 'Content-Type: application/json' \
+ -d '{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Answer with one word: what is 2+3?\"}]}]}' \
+ $$base/publishers/google/models/$(GEMINI_MODEL):generateContent \
+ | python3 -c 'import json,sys; d=json.load(sys.stdin); print(\" answer:\", d[\"candidates\"][0][\"content\"][\"parts\"][0][\"text\"].strip()) if \"candidates\" in d else print(\" refused:\", d)'; \
+ echo '== GET on the model, a method the grant does not list'; \
+ openshell sandbox exec --name $(NAME) --no-tty -- curl -sS -o /dev/null -w ' HTTP %{http_code} %header{Proxy-Status}\n' $$base/publishers/google/models/$(GEMINI_MODEL); \
+ echo '== POST to another Vertex AI path, outside the granted paths'; \
+ openshell sandbox exec --name $(NAME) --no-tty -- curl -sS -o /dev/null -w ' HTTP %{http_code} %header{Proxy-Status}\n' -X POST -d '{}' $$base/endpoints; \
+ if [ -n '$(AGENT_ENGINE)' ]; then \
+ echo '== POST reasoningEngines/$(AGENT_ENGINE):query, an agent on Agent Engine'; \
+ openshell sandbox exec --name $(NAME) --no-tty -- curl -sS --max-time 60 -H 'Content-Type: application/json' \
+ -d '{\"input\":{\"a\":40,\"b\":2}}' $$base/reasoningEngines/$(AGENT_ENGINE):query \
+ | python3 -c 'import json,sys; d=json.load(sys.stdin); print(\" output:\", json.dumps(d.get(\"output\", d)))'; \
+ fi"
+
+rogue:
+ $(MAKE) --no-print-directory sandbox NAME=rogue
+ @echo "== rogue has a SPIFFE ID but the mesh policy binds it to no role"
+ $(MAKE) --no-print-directory call NAME=rogue || true
+ @echo "== to let it in: make bind NAME=rogue"
+
+# OpenShell's guarantee that the supervisor is the only way out depends on the
+# CNI enforcing the NetworkPolicy it creates. From the workload pod, a direct
+# connection to the edge node must fail; through the supervisor it works.
+netpol-check:
+ id=$$(cat run/sandbox-id.$(NAME))
+ workload=$$($(K) -n openshell get pods -l openshell.ai/boundary-role=workload,openshell.ai/boundary-pair=$$id -o name | head -1)
+ test -n "$$workload" || { echo "no workload pod for sandbox $(NAME)"; exit 1; }
+ edge=$$($(K) -n agentmesh get svc edge-node -o jsonpath='{.spec.clusterIP}')
+ $(K) -n openshell get networkpolicy -o name
+ # By IP, so that the result is about the policy and not about DNS, which the
+ # policy also denies. kubectl exec runs outside the sandbox runtime's
+ # syscall interception, so this is the pod network the CNI sees.
+ if $(K) -n openshell exec $$workload -- timeout 5 curl -sS -o /dev/null http://$$edge:8080/healthz 2>/dev/null; then
+ echo "FAIL: $$workload reached the edge node at $$edge directly; the CNI does not enforce NetworkPolicy"; exit 1
+ fi
+ echo "OK: $$workload cannot reach $$edge:8080 directly (NetworkPolicy enforced)"
+ # Control: a pod in the same namespace that OpenShell's policies do not
+ # select reaches the same address, so the refusal above is enforcement and
+ # not a network outage. The functional proof is `make call`: the same
+ # request succeeds when it goes through the supervisor.
+ $(K) -n openshell run -q --rm -i --restart=Never netpol-control --image=curlimages/curl:8.10.1 -- \
+ -sS -o /dev/null -m 5 http://$$edge:8080/healthz \
+ || { echo "FAIL: an unconfined pod cannot reach the edge node either; this is a network problem, not enforcement"; exit 1; }
+ echo "OK: an unconfined pod in namespace openshell reaches $$edge:8080 (control)"
+
+policy:
+ if [ "$(HUB)" = local ]; then
+ $(K) -n agentmesh port-forward svc/agentmesh-one 18070:8080 >/dev/null 2>&1 &
+ pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT; sleep 2
+ fi
+ admin_token=$$(cat run/admin-token)
+ curl -sS -X POST "$(HUB_URL_LOCAL)/policies" \
+ -H "Authorization: Bearer $$admin_token" \
+ -H 'Content-Type: application/json' --data-binary @policy.json
+ @echo
+
+status:
+ $(K) -n spire get pods
+ $(K) -n agentmesh get pods
+ $(K) -n openshell get pods,sandboxes.agents.x-k8s.io,networkpolicies
+ $(MAKE) --no-print-directory _with-gateway-port-forward CMD='openshell sandbox list'
+ $(K) -n spire exec statefulset/spire-server -c spire-server -- /opt/spire/bin/spire-server entry show 2>/dev/null | grep -E "SPIFFE ID|Selector" | head -20
+
+down:
+ -$(MAKE) --no-print-directory _with-gateway-port-forward CMD='openshell sandbox delete demo rogue; openshell provider delete agentmesh; openshell profile delete agentmesh' 2>/dev/null
+ -openshell gateway remove $(OPENSHELL_GATEWAY) 2>/dev/null
+ -$(H) uninstall openshell -n openshell --wait
+ -$(K) delete namespace agentmesh --wait=false
+ -$(H) uninstall spire -n spire --wait
+ rm -rf run
+
+clean:
+ kind delete cluster --name $(KIND_CLUSTER_NAME)
+
+# Runs CMD with the OpenShell gateway port-forwarded to 127.0.0.1:18080.
+_with-gateway-port-forward:
+ @:
+ @if (exec 3<>/dev/tcp/127.0.0.1/$(GATEWAY_PORT)) 2>/dev/null; then
+ echo "127.0.0.1:$(GATEWAY_PORT) is in use (a port-forward left from an earlier run?); free it or set GATEWAY_PORT" >&2; exit 1
+ fi
+ @$(K) -n openshell port-forward svc/openshell $(GATEWAY_PORT):8080 >/dev/null 2>&1 &
+ @pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT INT TERM HUP
+ @for _ in $$(seq 1 20); do (exec 3<>/dev/tcp/127.0.0.1/$(GATEWAY_PORT)) 2>/dev/null && break; sleep 0.5; done
+ @# The CLI keeps one selected gateway; each target selects its own, so two
+ @# clusters (kind and GKE) can be driven from one workstation.
+ @openshell gateway select $(OPENSHELL_GATEWAY) >/dev/null 2>&1 || true
+ @$(CMD)
+
+run/join-token:
+ mkdir -p run && umask 077 && printf 'mesh_tok_%s' "$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" > $@
+
+run/admin-token:
+ mkdir -p run && umask 077 && printf 'mesh_adm_%s' "$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" > $@
+
+run/api-token:
+ mkdir -p run && umask 077 && head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n' > $@
diff --git a/development/examples/openshell-kubernetes/README.md b/development/examples/openshell-kubernetes/README.md
new file mode 100644
index 00000000..fd6dfbd9
--- /dev/null
+++ b/development/examples/openshell-kubernetes/README.md
@@ -0,0 +1,118 @@
+# An OpenShell sandbox on Kubernetes, on the mesh under its own identity
+
+This example is the Kubernetes counterpart of
+[`openshell-sandbox`](../openshell-sandbox/), the Docker driver example.
+OpenShell runs from its published Helm chart. Every sandbox gets a SPIFFE
+identity from SPIRE. The OpenShell supervisor beside each sandbox exchanges
+that identity for a mesh credential at the edge `agentmesh-node`. The agent
+in the sandbox holds no credential and sees the mesh as one unauthenticated
+MCP endpoint. To revoke a sandbox, change the policy.
+
+The example does not modify OpenShell. It installs the chart with one value
+(`provider_spiffe_workload_api_socket_path`). The integration is one
+provider profile.
+
+```text
+sandbox pod (agent, network off)
+ │ OpenShell Sandbox Protocol (mTLS, boundary Service)
+supervisor pod ── FetchJWTSVID ──▶ SPIRE agent (CSI socket)
+ │ POST /oauth/token (client_credentials + jwt-spiffe assertion)
+ │ X-Mesh-Authentication: Bearer <Biscuit>
+edge agentmesh-node ── /token/exchange ──▶ agentmesh-one (hub, in-cluster or Cloud Run)
+ │ mesh stream, caller's Biscuit
+office agentmesh-node ──▶ mcp://calc (allowed) / mcp://payroll (refused)
+```
+
+## What is different from the Docker example
+
+| | Docker driver | Kubernetes driver |
+|---|---|---|
+| Supervisor | container on the host network | Pod on the cluster network; the token endpoint and the MCP endpoint are the edge node's Service (`http://edge-node.agentmesh.svc.cluster.local:8080`), which OpenShell accepts as a plaintext in-cluster host |
+| Workload identity | SPIRE `docker` attestor, one entry per sandbox keyed on the `openshell.ai/sandbox-name` label | SPIRE Kubernetes stack (SPIFFE hardened charts); one `ClusterSPIFFEID` over supervisor pods using the `openshell.ai/sandbox-id` label, so the subject is `spiffe://openshell.local/openshell/sandbox/<uuid>`, NVIDIA's own template |
+| Binding | by name, known before creation | by sandbox id, known after creation (`make sandbox` prints it, `make bind` adds it); or the fleet with `user:spiffe://openshell.local/openshell/sandbox/*` |
+| Confinement | workload container with networking off | `agents.x-k8s.io` `Sandbox` + supervisor Pod + three `NetworkPolicy` objects; **the CNI must enforce them** (`make netpol-check` proves it) |
+| Operator access | `openshell gateway add http://127.0.0.1:8080` | chart-generated mTLS client bundle over a port-forward |
+
+## Run it
+
+You need `kubectl`, `helm`, `docker`, `kind` (for the local cluster), the
+`openshell` CLI (0.1.3 or later), `python3` and `curl`.
+
+```bash
+make cluster # a kind cluster
+make up # Agent Sandbox v1.0.7, SPIRE 0.29.0, the mesh (in-cluster hub), OpenShell 0.1.4
+make sandbox NAME=demo # prints spiffe://openshell.local/openshell/sandbox/<id>
+make call NAME=demo # refused: no role binds the identity yet
+make bind NAME=demo # adds the member, pushes the policy
+make call NAME=demo # calc allowed, payroll refused
+make netpol-check NAME=demo # workload pod cannot reach the edge node directly; an unconfined pod can
+make rogue # second sandbox, unbound, refused
+make gemini NAME=demo GCP_PROJECT=<project> # Gemini on Vertex AI through the mesh; see the guide
+make unbind NAME=demo # the same policy change in reverse; refused within about 3 minutes
+```
+
+Expected output of `make call` after `make bind`:
+
+```text
+== tools the mesh offers sandbox demo
+ 12D3KooWM5k2n5zm… mcp://calc/add Return a + b.
+ 12D3KooWM5k2n5zm… mcp://calc/multiply Return a * b.
+== mcp://calc, allowed for the sandbox role
+mcp://calc/add(2, 3) = 5.0
+== mcp://payroll, not in the sandbox role
+ (no tool of mcp://payroll is visible to this sandbox; calling the office node directly)
+mcp://payroll/add(2, 3) -> refused: auth rejected by 12D3KooWM5k2…: biscuit: denied by no matching policies
+```
+
+The hub log has one line per credential that the mesh minted:
+
+```text
+Border Crossing {"endpoint": "/token/exchange", "decision": "allow",
+ "subject": "spiffe://openshell.local/openshell/sandbox/2c399868-…", "roles": ["sandbox"]}
+```
+
+## On GKE, with the hub on Cloud Run
+
+The [OpenShell and Substrate guide](../../../site/content/docs/guides/openshell-and-substrate.md)
+has the full procedure. A summary:
+
+- Create the node pool for the sandboxes with `--image-type UBUNTU_CONTAINERD`.
+ The OpenShell sandbox runtime needs Landlock ABI v3. The
+ Container-Optimized OS kernel has no `CONFIG_SECURITY_LANDLOCK`.
+- Use Dataplane V2. It enforces `NetworkPolicy`. `make netpol-check` shows
+ this.
+- GKE assigns ClusterIPs outside RFC 1918. The example renders `allowed_ips`
+ in the provider profile from the ClusterIP of the edge Service.
+- Expose the SPIRE OIDC discovery provider on a reserved address. Name that
+ address as the issuer: `make prereqs OIDC_SERVICE_TYPE=LoadBalancer
+ OIDC_LOAD_BALANCER_IP=<ip> JWT_ISSUER=http://<ip>`. Pass the issuer to the
+ hub in `--workload-issuer`.
+- Set `KUBE_CONTEXT` to the cluster and `HUB` to the URL of the hub. Set
+ `CALC_MCP_IMAGE` and `SANDBOX_IMAGE` to images that the cluster can pull.
+ Put the join token and the admin token of the hub in `run/join-token` and
+ `run/admin-token`.
+
+```bash
+export KUBE_CONTEXT=gke_<project>_<zone>_<cluster> HUB=https://<hub>.run.app OPENSHELL_GATEWAY=openshell-gke
+make mesh CALC_MCP_IMAGE=<registry>/calc-mcp:v1
+make openshell SANDBOX_IMAGE=<registry>/sandbox:v1
+make sandbox NAME=demo && make bind NAME=demo && make call NAME=demo
+```
+
+`make bind` and `make policy` push the policy to the hub with the admin
+token. If the hub mounts the policy from Secret Manager, also add the new
+policy as a secret version. Then the next revision starts with it.
+
+## Files
+
+- `spire/values.yaml.tmpl`: SPIFFE hardened chart values; trust domain,
+ issuer, the `ClusterSPIFFEID`s for supervisors and the gateway, OIDC provider
+ over plain HTTP in-cluster.
+- `openshell/values.yaml`: the one OpenShell chart value.
+- `openshell/provider-profile.yaml`: the `token_grant` profile.
+- `mesh/hub.yaml.tmpl`: in-cluster `agentmesh-one` (only with `HUB=local`).
+- `mesh/nodes.yaml.tmpl`: edge node, office node, calculator.
+- `policy.json`: the mesh policy; `bind.py` edits its `sandbox` binding.
+- `mcp.py`: runs `curl` inside the sandbox through `openshell sandbox exec`.
+- `Dockerfile.sandbox`: the sandbox image (Ubuntu + curl); an agent image can
+ replace it.
diff --git a/development/examples/openshell-kubernetes/bind.py b/development/examples/openshell-kubernetes/bind.py
new file mode 100644
index 00000000..39b6e7be
--- /dev/null
+++ b/development/examples/openshell-kubernetes/bind.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+# Adds a sandbox's SPIFFE identity to the sandbox role in policy.json.
+#
+# bind.py <member> e.g. user:spiffe://openshell.local/openshell/sandbox/<id>
+# bind.py --remove <member>
+import json
+import sys
+
+args = sys.argv[1:]
+remove = False
+if args and args[0] == "--remove":
+ remove = True
+ args = args[1:]
+if len(args) != 1:
+ sys.exit(__doc__)
+member = args[0]
+
+with open("policy.json") as f:
+ policy = json.load(f)
+binding = next((b for b in policy["bindings"] if b["role"] == "sandbox"), None)
+if binding is None:
+ binding = {"role": "sandbox", "members": []}
+ policy["bindings"].append(binding)
+if remove:
+ binding["members"] = [m for m in binding["members"] if m != member]
+ if not binding["members"]:
+ policy["bindings"].remove(binding)
+ print("unbound", member)
+elif member not in binding["members"]:
+ binding["members"].append(member)
+ print("bound", member)
+else:
+ print("already bound", member)
+with open("policy.json", "w") as f:
+ json.dump(policy, f, indent=2)
+ f.write("\n")
diff --git a/development/examples/openshell-kubernetes/mcp.py b/development/examples/openshell-kubernetes/mcp.py
new file mode 100644
index 00000000..0b5b7d53
--- /dev/null
+++ b/development/examples/openshell-kubernetes/mcp.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# Calls the edge node's MCP server from inside an OpenShell sandbox, the way an
+# agent would: plain HTTP to the edge node Service, no credential in hand.
+# The OpenShell supervisor adds the sandbox's mesh credential on the way out.
+#
+# mcp.py tools SANDBOX [SERVICE] tools the mesh offers this sandbox
+# mcp.py add SANDBOX SERVICE A B call mcp://SERVICE/add through the mesh
+import json
+import subprocess
+import os
+import sys
+
+NODE = "http://edge-node.agentmesh.svc.cluster.local:8080/mcp"
+
+
+def rpc(sandbox, tool, arguments):
+ body = json.dumps({
+ "jsonrpc": "2.0",
+ "id": 1,
+ "method": "tools/call",
+ "params": {"name": tool, "arguments": arguments},
+ })
+ out = subprocess.run(
+ ["openshell", "sandbox", "exec", "--name", sandbox, "--no-tty", "--",
+ "curl", "-sS", "--max-time", "60", "-w", "\n%{http_code}", NODE,
+ "-H", "Content-Type: application/json",
+ "-H", "Accept: application/json, text/event-stream",
+ # OpenShell inspects MCP and wants the revision stated, as a real
+ # client does after initialize.
+ "-H", "MCP-Protocol-Version: 2025-11-25",
+ "-d", body],
+ check=True, capture_output=True, text=True,
+ ).stdout
+ out, _, status = out.rpartition("\n")
+ # Streamable HTTP answers as JSON or as one SSE message.
+ payload = out
+ if out.lstrip().startswith("event:") or "\ndata: " in out:
+ payload = "".join(line[6:] for line in out.splitlines() if line.startswith("data: "))
+ try:
+ reply = json.loads(payload)
+ except ValueError:
+ sys.exit(f"{tool}: HTTP {status}: {out.strip()[:300]}")
+ if "error" in reply:
+ error = reply["error"]
+ sys.exit(f"{tool}: HTTP {status}: {error.get('message', error) if isinstance(error, dict) else error}")
+ result = reply["result"]
+ text = "\n".join(c.get("text", "") for c in result.get("content", []) if c.get("type") == "text")
+ if result.get("isError"):
+ return None, text
+ try:
+ return json.loads(text), text
+ except ValueError:
+ return text, text
+
+
+def tools(sandbox, service=None):
+ args = {"service_name": service} if service else {}
+ rows, text = rpc(sandbox, "find_remote_tools", args)
+ if rows is None:
+ sys.exit(f"find_remote_tools refused: {text}")
+ if isinstance(rows, dict):
+ rows = rows.get("tools") or rows.get("results") or [rows]
+ for row in rows:
+ if "error" in row:
+ print(f" {row.get('peer_id', '?')[:16]}… {row['error']}")
+ else:
+ print(f" {row['peer_id'][:16]}… {row['tool_name']:<24} {row.get('description', '')}")
+ return rows
+
+
+def add(sandbox, service, a, b):
+ rows = tools(sandbox, service)
+ peers = {r["peer_id"] for r in rows if r.get("tool_name") == f"mcp://{service}/add"}
+ if not peers:
+ # Discovery already hides what the policy refuses. Ask a peer anyway, so
+ # the refusal itself is visible.
+ print(f" (no tool of mcp://{service} is visible to this sandbox; calling the office node directly)")
+ peers = {r["peer_id"] for r in rpc(sandbox, "discover_remote_services", {"type": "mcp"})[0] or []}
+ if not peers:
+ print(f"mcp://{service}/add({a}, {b}) -> refused: the mesh shows this sandbox no MCP provider at all (no role binds its identity)")
+ sys.exit(3)
+ # PEER=<prefix> picks a provider when several serve the tool, for example
+ # the one in another cluster; the default is the first by peer id.
+ want = os.environ.get("PEER", "")
+ chosen = sorted(p for p in peers if p.startswith(want)) or sorted(peers)
+ peer = chosen[0]
+ print(f" calling provider {peer[:16]}…")
+ result, text = rpc(sandbox, "call_remote_tool", {
+ "peer_id": peer,
+ "tool_name": f"mcp://{service}/add",
+ "arguments": {"a": float(a), "b": float(b)},
+ })
+ if result is None:
+ print(f"mcp://{service}/add({a}, {b}) -> refused: {text}")
+ sys.exit(3)
+ print(f"mcp://{service}/add({a}, {b}) = {text}")
+
+
+if __name__ == "__main__":
+ cmd, *rest = sys.argv[1:] or ["help"]
+ if cmd == "tools" and 1 <= len(rest) <= 2:
+ tools(*rest)
+ elif cmd == "add" and len(rest) == 4:
+ add(*rest)
+ else:
+ sys.exit(__doc__ or "usage: mcp.py tools SANDBOX [SERVICE] | add SANDBOX SERVICE A B")
diff --git a/development/examples/openshell-kubernetes/mesh/hub.yaml.tmpl b/development/examples/openshell-kubernetes/mesh/hub.yaml.tmpl
new file mode 100644
index 00000000..9f638925
--- /dev/null
+++ b/development/examples/openshell-kubernetes/mesh/hub.yaml.tmpl
@@ -0,0 +1,65 @@
+# agentmesh-one in the cluster: control plane, router and console on one port.
+# Used when HUB=local (the default on kind). With HUB=<url> the nodes join an
+# agentmesh-one running elsewhere, such as Cloud Run, and this file is not
+# applied.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: hub-config
+ namespace: agentmesh
+data:
+ policy.json: |
+${POLICY_JSON_INDENTED}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: agentmesh-one
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: agentmesh-one}
+ template:
+ metadata:
+ labels: {app: agentmesh-one}
+ spec:
+ containers:
+ - name: agentmesh-one
+ image: ${AGENTMESH_ONE_IMAGE}
+ args:
+ - --bind-address=0.0.0.0
+ - --port=8080
+ - --data-dir=/data
+ - --policy-file=/etc/agentmesh/policy.json
+ - --token-path=/run/agentmesh/join-token
+ - --admin-token-path=/run/agentmesh/admin-token
+ # SPIRE's OIDC discovery provider is the workload identity issuer; a
+ # JWT-SVID from it with aud=agentmesh is an identity the control plane
+ # accepts at /token/exchange and binds roles to.
+ - --workload-issuer=${JWT_ISSUER}
+ - --allowed-audiences=agentmesh
+ - --enroll-qr=false
+ ports:
+ - {name: http, containerPort: 8080}
+ volumeMounts:
+ - {name: config, mountPath: /etc/agentmesh, readOnly: true}
+ - {name: tokens, mountPath: /run/agentmesh, readOnly: true}
+ - {name: data, mountPath: /data}
+ volumes:
+ - name: config
+ configMap: {name: hub-config}
+ - name: tokens
+ secret: {secretName: agentmesh-tokens, defaultMode: 0444}
+ - name: data
+ emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: agentmesh-one
+ namespace: agentmesh
+spec:
+ selector: {app: agentmesh-one}
+ ports:
+ - {name: http, port: 8080, targetPort: http}
diff --git a/development/examples/openshell-kubernetes/mesh/nodes.yaml.tmpl b/development/examples/openshell-kubernetes/mesh/nodes.yaml.tmpl
new file mode 100644
index 00000000..8bce6a42
--- /dev/null
+++ b/development/examples/openshell-kubernetes/mesh/nodes.yaml.tmpl
@@ -0,0 +1,145 @@
+# The mesh beside OpenShell, in one namespace: an edge node that is the
+# sandboxes' door to the mesh and the supervisors' token endpoint, an office
+# node fronting a calculator, and the calculator. The control plane is either
+# agentmesh-one in this cluster (hub.yaml.tmpl) or an agentmesh-one elsewhere,
+# such as Cloud Run; CONTROL_PLANE_URL names it.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: mesh-config
+ namespace: agentmesh
+data:
+ office.yaml: |
+ version: "v1alpha1"
+ labels:
+ site: office
+ services:
+ - type: mcp
+ name: calc
+ description: "calculator MCP server in the office"
+ target_url: "http://calc-mcp.agentmesh.svc.cluster.local:7777/mcp"
+ - type: mcp
+ name: payroll
+ description: "an MCP server the sandbox role may not call"
+ target_url: "http://calc-mcp.agentmesh.svc.cluster.local:7777/mcp"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: edge-node
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: edge-node}
+ template:
+ metadata:
+ labels: {app: edge-node}
+ spec:
+ containers:
+ - name: agentmesh-node
+ image: ${AGENTMESH_NODE_IMAGE}
+ args:
+ - run
+ - --control-plane=${CONTROL_PLANE_URL}
+ ${CONTROL_PLANE_INSECURE_ARG}
+ - --bootstrap-token-path=/run/agentmesh/join-token
+ - --api-token-path=/run/agentmesh/api-token
+ - --data-dir=/data
+ - --bind-addr=0.0.0.0:8080
+ - --control-plane-sync-interval=30s
+ - --metrics-addr=0.0.0.0:9090
+ ports:
+ - {name: http, containerPort: 8080}
+ readinessProbe:
+ httpGet: {path: /readyz, port: 9090}
+ periodSeconds: 2
+ volumeMounts:
+ - {name: tokens, mountPath: /run/agentmesh, readOnly: true}
+ - {name: data, mountPath: /data}
+ volumes:
+ - name: tokens
+ secret: {secretName: agentmesh-tokens, defaultMode: 0444}
+ - name: data
+ emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: edge-node
+ namespace: agentmesh
+spec:
+ selector: {app: edge-node}
+ ports:
+ - {name: http, port: 8080, targetPort: http}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: office-node
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: office-node}
+ template:
+ metadata:
+ labels: {app: office-node}
+ spec:
+ containers:
+ - name: agentmesh-node
+ image: ${AGENTMESH_NODE_IMAGE}
+ args:
+ - run
+ - --control-plane=${CONTROL_PLANE_URL}
+ ${CONTROL_PLANE_INSECURE_ARG}
+ - --bootstrap-token-path=/run/agentmesh/join-token
+ - --api-token-path=/run/agentmesh/api-token
+ - --data-dir=/data
+ - --bind-addr=
+ - --config=/etc/agentmesh/office.yaml
+ - --control-plane-sync-interval=30s
+ - --metrics-addr=0.0.0.0:9090
+ readinessProbe:
+ httpGet: {path: /readyz, port: 9090}
+ periodSeconds: 2
+ volumeMounts:
+ - {name: config, mountPath: /etc/agentmesh, readOnly: true}
+ - {name: tokens, mountPath: /run/agentmesh, readOnly: true}
+ - {name: data, mountPath: /data}
+ volumes:
+ - name: config
+ configMap: {name: mesh-config}
+ - name: tokens
+ secret: {secretName: agentmesh-tokens, defaultMode: 0444}
+ - name: data
+ emptyDir: {}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: calc-mcp
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: calc-mcp}
+ template:
+ metadata:
+ labels: {app: calc-mcp}
+ spec:
+ containers:
+ - name: calc-mcp
+ image: ${CALC_MCP_IMAGE}
+ ports:
+ - {name: http, containerPort: 7777}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: calc-mcp
+ namespace: agentmesh
+spec:
+ selector: {app: calc-mcp}
+ ports:
+ - {name: http, port: 7777, targetPort: http}
diff --git a/development/examples/openshell-kubernetes/openshell/provider-profile.yaml.tmpl b/development/examples/openshell-kubernetes/openshell/provider-profile.yaml.tmpl
new file mode 100644
index 00000000..b253faea
--- /dev/null
+++ b/development/examples/openshell-kubernetes/openshell/provider-profile.yaml.tmpl
@@ -0,0 +1,67 @@
+# OpenShell provider profile for the Agent Mesh edge node running in the
+# cluster. The credential is not stored anywhere: for each sandbox the
+# supervisor (a pod on the cluster network) trades the sandbox's JWT-SVID at the
+# node's token endpoint (RFC 7523 client assertion) for a Biscuit bound to that
+# identity, caches it, and injects it as X-Mesh-Authentication on every request
+# the sandbox makes to the node. OpenShell accepts a plaintext token endpoint
+# only on loopback or on an in-cluster Service host, which this is.
+#
+# allowed_ips pins the endpoint to the edge node Service's ClusterIP, which the
+# Makefile reads from the cluster: the supervisor refuses a name that resolves
+# outside this list (policy_dns_no_valid_address), and Service CIDRs differ per
+# cluster (kind 10.96.0.0/16, GKE 34.118.224.0/20).
+id: agentmesh
+display_name: Agent Mesh
+description: Mesh services and models through the cluster's agentmesh-node, under the sandbox's own identity
+category: other
+
+credentials:
+ - name: mesh_token
+ description: Biscuit the mesh issues for this sandbox's SPIFFE identity
+ required: false
+ auth_style: bearer
+ header_name: X-Mesh-Authentication
+ token_grant:
+ grant_type: client_credentials
+ token_endpoint: http://edge-node.agentmesh.svc.cluster.local:8080/oauth/token
+ # The aud claim of the JWT-SVID; the control plane accepts it with
+ # --allowed-audiences.
+ jwt_svid_audience: agentmesh
+ client_assertion_type: urn:ietf:params:oauth:client-assertion-type:jwt-spiffe
+ # How long the supervisor keeps a Biscuit before trading a fresh SVID for
+ # a new one; a policy change reaches a sandbox within this time.
+ cache_ttl_seconds: 60
+
+endpoints:
+ # The node's MCP server: discover and call tools across the mesh.
+ - host: edge-node.agentmesh.svc.cluster.local
+ port: 8080
+ path: /mcp
+ protocol: mcp
+ enforcement: enforce
+ allowed_ips: [${EDGE_NODE_IP}/32]
+ mcp:
+ allow_all_known_mcp_methods: true
+ rules:
+ - allow: {}
+ # The node's OpenAI-compatible endpoint for models published on the mesh.
+ - host: edge-node.agentmesh.svc.cluster.local
+ port: 8080
+ path: /v1/**
+ protocol: rest
+ access: read-write
+ enforcement: enforce
+ allowed_ips: [${EDGE_NODE_IP}/32]
+ # Services another node serves, reached through this node by peer id:
+ # /mesh/<peer>/egress/<destination>/... for an egress destination. Which
+ # destinations, methods and paths the sandbox may use is the mesh policy.
+ - host: edge-node.agentmesh.svc.cluster.local
+ port: 8080
+ path: /mesh/**
+ protocol: rest
+ access: read-write
+ enforcement: enforce
+ allowed_ips: [${EDGE_NODE_IP}/32]
+
+binaries:
+ - /**
diff --git a/development/examples/openshell-kubernetes/openshell/values.yaml b/development/examples/openshell-kubernetes/openshell/values.yaml
new file mode 100644
index 00000000..ec05c205
--- /dev/null
+++ b/development/examples/openshell-kubernetes/openshell/values.yaml
@@ -0,0 +1,11 @@
+# OpenShell Helm chart values for the Agent Mesh walkthrough. Everything is the
+# chart's default except the one setting the integration needs: the SPIFFE
+# Workload API socket the Kubernetes driver projects into the gateway pod and
+# into every sandbox supervisor pod, so that supervisors can ask SPIRE for a
+# JWT-SVID when a provider profile declares a token grant.
+#
+# The chart's PKI job generates the gateway's server/client mTLS material; the
+# CLI uses the client bundle over a port-forward (see the Makefile).
+gatewayConfig:
+ openshell.drivers.kubernetes:
+ provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock
diff --git a/development/examples/openshell-kubernetes/policy.json b/development/examples/openshell-kubernetes/policy.json
new file mode 100644
index 00000000..ba5a8f42
--- /dev/null
+++ b/development/examples/openshell-kubernetes/policy.json
@@ -0,0 +1,39 @@
+{
+ "roles": [
+ {
+ "name": "mesh:role:node",
+ "allowed_services": [
+ "*"
+ ],
+ "allowed_publish": [
+ "*"
+ ],
+ "allowed_targets": [
+ "*"
+ ],
+ "allowed_labels": [
+ "*"
+ ]
+ },
+ {
+ "name": "mesh:role:router",
+ "allowed_services": [
+ "*"
+ ],
+ "allowed_targets": [
+ "*"
+ ]
+ },
+ {
+ "name": "sandbox",
+ "allowed_services": [
+ "system://mesh.catalog",
+ "mcp://calc"
+ ],
+ "allowed_targets": [
+ "*"
+ ]
+ }
+ ],
+ "bindings": []
+}
diff --git a/development/examples/openshell-kubernetes/profile-apply.sh b/development/examples/openshell-kubernetes/profile-apply.sh
new file mode 100755
index 00000000..e6b14bab
--- /dev/null
+++ b/development/examples/openshell-kubernetes/profile-apply.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Imports the provider profile into the active OpenShell gateway, or updates it
+# in place when it already exists (an update must carry the stored
+# resource_version).
+#
+# profile-apply.sh <profile.yaml>
+set -euo pipefail
+
+file=$1
+id=$(awk '/^id:/ {print $2; exit}' "$file")
+rv=$(openshell profile list -o json 2>/dev/null | python3 -c '
+import json, sys
+d = json.load(sys.stdin)
+d = d if isinstance(d, list) else d.get("profiles", [])
+print(next((p["resource_version"] for p in d if p.get("id") == sys.argv[1]), 0))' "$id")
+if [ "$rv" = 0 ]; then
+ openshell profile import -f "$file"
+else
+ tmp=$(mktemp --suffix=.yaml)
+ trap 'rm -f "$tmp"' EXIT
+ sed "s/^id: $id\$/id: $id\nresource_version: $rv/" "$file" > "$tmp"
+ openshell profile update -f "$tmp" "$id"
+fi
diff --git a/development/examples/openshell-kubernetes/spire/values.yaml.tmpl b/development/examples/openshell-kubernetes/spire/values.yaml.tmpl
new file mode 100644
index 00000000..22e44560
--- /dev/null
+++ b/development/examples/openshell-kubernetes/spire/values.yaml.tmpl
@@ -0,0 +1,55 @@
+# SPIRE for OpenShell sandboxes on Kubernetes (SPIFFE hardened chart `spire`).
+#
+# - trust domain openshell.local; the OIDC discovery provider serves plain HTTP
+# in-cluster (tls.spire.enabled=false puts an nginx in front of it on :80),
+# which the mesh control plane names with --workload-issuer.
+# - the CSI driver mounts the Workload API socket into any pod that declares a
+# csi.spiffe.io volume; OpenShell's gateway and every sandbox supervisor do
+# when provider_spiffe_workload_api_socket_path is set.
+# - one ClusterSPIFFEID gives every sandbox supervisor an identity derived from
+# the sandbox id OpenShell puts on the supervisor pod. This is NVIDIA's own
+# template from deploy/helm/openshell/ci/values-spire-stack.yaml, narrowed to
+# supervisor pods; a second entry identifies the gateway pod so it can fetch
+# JWT bundles to verify supervisors.
+global:
+ spire:
+ clusterName: ${CLUSTER_NAME}
+ trustDomain: openshell.local
+ jwtIssuer: ${JWT_ISSUER}
+
+spire-server:
+ defaultJwtSvidTTL: 5m
+ controllerManager:
+ identities:
+ clusterSPIFFEIDs:
+ default:
+ enabled: false
+ openshell-sandboxes:
+ enabled: true
+ spiffeIDTemplate: 'spiffe://{{ .TrustDomain }}/openshell/sandbox/{{ index .PodMeta.Labels "openshell.ai/sandbox-id" }}'
+ namespaceSelector:
+ matchLabels:
+ kubernetes.io/metadata.name: openshell
+ podSelector:
+ matchLabels:
+ openshell.ai/managed-by: openshell
+ openshell.ai/component: supervisor
+ openshell-gateway:
+ enabled: true
+ spiffeIDTemplate: 'spiffe://{{ .TrustDomain }}/openshell/gateway'
+ namespaceSelector:
+ matchLabels:
+ kubernetes.io/metadata.name: openshell
+ podSelector:
+ matchLabels:
+ app.kubernetes.io/name: openshell
+ app.kubernetes.io/instance: openshell
+
+spiffe-oidc-discovery-provider:
+ enabled: true
+ tls:
+ spire:
+ enabled: false
+ service:
+ type: ${OIDC_SERVICE_TYPE}
+ loadBalancerIP: "${OIDC_LOAD_BALANCER_IP}"
diff --git a/development/examples/openshell-sandbox/Makefile b/development/examples/openshell-sandbox/Makefile
index b06a8e29..5a9c4d91 100644
--- a/development/examples/openshell-sandbox/Makefile
+++ b/development/examples/openshell-sandbox/Makefile
@@ -20,7 +20,7 @@ TRUST_DOMAIN := openshell.local
SPIRE := docker compose exec -T --interactive=false spire-server /opt/spire/bin/spire-server
SANDBOX_IMAGE := openshell-agentmesh-sandbox:local
-.PHONY: up sandbox call rogue policy status down
+.PHONY: up sandbox call rogue policy status down select-gateway
up: run/join-token run/api-token run/admin-token
@command -v openshell >/dev/null || { echo "openshell CLI not found: https://docs.openshell.ai/about/installation"; exit 1; }
@@ -39,6 +39,7 @@ up: run/join-token run/api-token run/admin-token
@for _ in $$(seq 1 60); do curl -sf http://127.0.0.1:8085/healthz >/dev/null && break; sleep 1; done
curl -sf http://127.0.0.1:8085/healthz >/dev/null
openshell gateway list 2>/dev/null | grep -q '^$(GATEWAY)\b' || openshell gateway add http://127.0.0.1:8080 --name $(GATEWAY)
+ openshell gateway select $(GATEWAY) >/dev/null
openshell profile import -f openshell/provider-profile.yaml
openshell provider list 2>/dev/null | grep -q '^agentmesh\b' || openshell provider create --name agentmesh --type agentmesh --runtime-credentials
@echo
@@ -48,7 +49,10 @@ up: run/join-token run/api-token run/admin-token
# The sandbox's identity is a SPIRE registration entry keyed on the label
# OpenShell puts on the sandbox's supervisor container. The mesh policy
# (policy.json) says what that identity may call.
-sandbox:
+# The CLI keeps one selected gateway; the targets that talk to this one
+# select it, so another cluster's gateway on the same workstation does not
+# receive the commands.
+sandbox: select-gateway
entry=$$($(SPIRE) entry show -spiffeID spiffe://$(TRUST_DOMAIN)/sandbox/$(NAME) | awk '/^Entry ID/ {print $$4}')
if [ -z "$$entry" ]; then
entry=$$($(SPIRE) entry create \
@@ -65,7 +69,7 @@ sandbox:
openshell sandbox create --name $(NAME) --from $(SANDBOX_IMAGE) --provider agentmesh --detach -- sleep infinity
@echo "sandbox $(NAME) is spiffe://$(TRUST_DOMAIN)/sandbox/$(NAME) (SPIRE entry $$entry); try: make call NAME=$(NAME)"
-call:
+call: select-gateway
@echo "== tools the mesh offers sandbox $(NAME)"
python3 mcp.py tools $(NAME)
@echo "== mcp://calc, allowed for the sandbox role"
@@ -110,3 +114,6 @@ run/admin-token:
run/api-token:
mkdir -p run && umask 022 && head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n' > $@
+
+select-gateway:
+ @openshell gateway select $(GATEWAY) >/dev/null
diff --git a/development/examples/openshell-sandbox/compose.yaml b/development/examples/openshell-sandbox/compose.yaml
index a72380b5..eca79d20 100644
--- a/development/examples/openshell-sandbox/compose.yaml
+++ b/development/examples/openshell-sandbox/compose.yaml
@@ -12,7 +12,7 @@
name: openshell-agentmesh
x-agentmesh-node: &agentmesh-node
- image: ${AGENTMESH_NODE_IMAGE:-ghcr.io/google/agentmesh-node:${AGENTMESH_VERSION:-latest}}
+ image: ${AGENTMESH_NODE_IMAGE:-ghcr.io/google/agentmesh-node:${AGENTMESH_VERSION:-v0.1.1}}
restart: unless-stopped
depends_on:
agentmesh-one:
@@ -64,7 +64,7 @@ services:
# ----------------------------------------------------------- Agent Mesh ----
agentmesh-one:
- image: ${AGENTMESH_ONE_IMAGE:-ghcr.io/google/agentmesh-one:${AGENTMESH_VERSION:-latest}}
+ image: ${AGENTMESH_ONE_IMAGE:-ghcr.io/google/agentmesh-one:${AGENTMESH_VERSION:-v0.1.1}}
restart: unless-stopped
command:
- --bind-address=0.0.0.0
diff --git a/development/examples/openshell-sandbox/policy.json b/development/examples/openshell-sandbox/policy.json
index b4d2320b..96a9286b 100644
--- a/development/examples/openshell-sandbox/policy.json
+++ b/development/examples/openshell-sandbox/policy.json
@@ -3,6 +3,7 @@
{
"name": "mesh:role:node",
"allowed_services": ["*"],
+ "allowed_publish": ["*"],
"allowed_targets": ["*"],
"allowed_labels": ["*"]
},
diff --git a/development/examples/substrate-actor/.gitignore b/development/examples/substrate-actor/.gitignore
new file mode 100644
index 00000000..7e3b24e0
--- /dev/null
+++ b/development/examples/substrate-actor/.gitignore
@@ -0,0 +1 @@
+run/
diff --git a/development/examples/substrate-actor/Makefile b/development/examples/substrate-actor/Makefile
index 87ca6f4c..654cc594 100644
--- a/development/examples/substrate-actor/Makefile
+++ b/development/examples/substrate-actor/Makefile
@@ -21,7 +21,11 @@ SHELL := /bin/bash
NAME ?= demo
SUBSTRATE_DIR ?= $(abspath ../../../../substrate)
KIND_CLUSTER_NAME ?= kind
-CTX := kind-$(KIND_CLUSTER_NAME)
+# KUBE_CONTEXT selects the cluster. The default is the kind cluster `make
+# cluster` creates; for a cluster Substrate's own tooling set up (GKE through
+# `tools/setup-gcp bootstrap` + `hack/install-ate.sh`), point it there.
+KUBE_CONTEXT ?= kind-$(KIND_CLUSTER_NAME)
+CTX := $(KUBE_CONTEXT)
KUBECTL := kubectl --context $(CTX)
KATE := kubectl-ate --context $(CTX)
ATESPACE := ate-demo-sandbox
@@ -29,15 +33,42 @@ TEMPLATE := sandbox-template
TRUST_DOMAIN := substrate-actor.local
ROUTER_PORT ?= 8000
KO_DOCKER_REPO ?= localhost:5001
+# CALC_MCP_IMAGE: a pullable image for the calc MCP server. Empty builds
+# ../calc-mcp and loads it into the kind cluster; on any other cluster set it
+# to an image the nodes can pull.
+CALC_MCP_IMAGE ?=
-AGENTMESH_VERSION ?= latest
+AGENTMESH_VERSION ?= v0.1.1
AGENTMESH_ONE_IMAGE ?= ghcr.io/google/agentmesh-one:$(AGENTMESH_VERSION)
AGENTMESH_NODE_IMAGE ?= ghcr.io/google/agentmesh-node:$(AGENTMESH_VERSION)
+# HUB=local runs agentmesh-one in the cluster; HUB=https://... joins one that
+# runs elsewhere (Cloud Run). With a remote hub the provider's issuer must be
+# reachable from it: give the provider a persistent key (PROVIDER_KEY_PATH)
+# and publish its discovery document and JWKS where ISSUER points (a public
+# bucket), see README. run/join-token and run/admin-token hold the hub's
+# tokens when HUB is remote.
+HUB ?= local
+ifeq ($(HUB),local)
+CONTROL_PLANE_URL := http://agentmesh-one.agentmesh.svc.cluster.local:8080
+CONTROL_PLANE_INSECURE_ARG := - --insecure-control-plane
+ISSUER ?= http://credential-provider.agentmesh.svc.cluster.local:8080
+else
+CONTROL_PLANE_URL := $(HUB)
+CONTROL_PLANE_INSECURE_ARG :=
+ISSUER ?= $(error ISSUER is required with a remote HUB: the public URL the provider's discovery document is published at)
+endif
+PROVIDER_KEY_PATH ?=
+ifneq ($(PROVIDER_KEY_PATH),)
+PROVIDER_KEY_ARGS := - --signing-key-path=/run/agentmesh-provider/p256.pem\n - --jwks-uri=$(ISSUER)/keys
+else
+PROVIDER_KEY_ARGS :=
+endif
+
KUBECTL_CONTEXT := $(CTX)
export KIND_CLUSTER_NAME KUBECTL_CONTEXT KO_DOCKER_REPO
-.PHONY: cluster up actor call rogue policy status down clean
+.PHONY: cluster up issuer-docs actor call gemini rogue policy status down clean
cluster:
@for t in docker kind ko kubectl go python3; do command -v $$t >/dev/null || { echo "$$t not found"; exit 1; }; done
@@ -51,25 +82,48 @@ cluster:
cd "$(SUBSTRATE_DIR)" && go install ./cmd/kubectl-ate
@echo "Substrate is up; next: make up"
-up:
+up: run/join-token run/admin-token run/api-token
$(KUBECTL) apply -f manifests/namespace.yaml
$(KUBECTL) -n agentmesh get secret agentmesh-tokens >/dev/null 2>&1 || \
$(KUBECTL) -n agentmesh create secret generic agentmesh-tokens \
- --from-literal=join-token="mesh_tok_$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" \
- --from-literal=admin-token="mesh_adm_$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" \
- --from-literal=api-token="$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')"
+ --from-file=join-token=run/join-token --from-file=admin-token=run/admin-token --from-file=api-token=run/api-token
+ if [ -n "$(PROVIDER_KEY_PATH)" ]; then
+ $(KUBECTL) -n agentmesh create secret generic credential-provider-signing-key --from-file=p256.pem=$(PROVIDER_KEY_PATH) --dry-run=client -o yaml | $(KUBECTL) apply -f -
+ fi
provider=$$(cd credential-provider && KO_DOCKER_REPO=$(KO_DOCKER_REPO)/agentmesh-credential-provider ko build --platform=linux/$$(go env GOARCH) --bare .)
- calc=$$(docker build -q ../calc-mcp -t calc-mcp:local >/dev/null && kind load docker-image calc-mcp:local --name $(KIND_CLUSTER_NAME) >/dev/null && echo docker.io/library/calc-mcp:local)
- for f in manifests/mesh.yaml.tmpl manifests/credential-provider.yaml.tmpl; do
- AGENTMESH_ONE_IMAGE="$(AGENTMESH_ONE_IMAGE)" AGENTMESH_NODE_IMAGE="$(AGENTMESH_NODE_IMAGE)" \
- CREDENTIAL_PROVIDER_IMAGE="$$provider" CALC_MCP_IMAGE="$$calc" \
- envsubst '$$AGENTMESH_ONE_IMAGE $$AGENTMESH_NODE_IMAGE $$CREDENTIAL_PROVIDER_IMAGE $$CALC_MCP_IMAGE' < $$f | $(KUBECTL) apply -f -
- done
+ calc="$(CALC_MCP_IMAGE)"
+ if [ -z "$$calc" ]; then
+ calc=$$(docker build -q ../calc-mcp -t calc-mcp:local >/dev/null && kind load docker-image calc-mcp:local --name $(KIND_CLUSTER_NAME) >/dev/null && echo docker.io/library/calc-mcp:local)
+ fi
+ if [ "$(HUB)" = local ]; then
+ AGENTMESH_ONE_IMAGE="$(AGENTMESH_ONE_IMAGE)" ISSUER="$(ISSUER)" envsubst '$$AGENTMESH_ONE_IMAGE $$ISSUER' < manifests/hub.yaml.tmpl | $(KUBECTL) apply -f -
+ fi
+ AGENTMESH_NODE_IMAGE="$(AGENTMESH_NODE_IMAGE)" CALC_MCP_IMAGE="$$calc" CONTROL_PLANE_URL="$(CONTROL_PLANE_URL)" CONTROL_PLANE_INSECURE_ARG="$(CONTROL_PLANE_INSECURE_ARG)" \
+ envsubst '$$AGENTMESH_NODE_IMAGE $$CALC_MCP_IMAGE $$CONTROL_PLANE_URL $$CONTROL_PLANE_INSECURE_ARG' < manifests/mesh.yaml.tmpl | $(KUBECTL) apply -f -
+ CREDENTIAL_PROVIDER_IMAGE="$$provider" ISSUER="$(ISSUER)" PROVIDER_KEY_ARGS="$$(printf '%b' '$(PROVIDER_KEY_ARGS)')" \
+ envsubst '$$CREDENTIAL_PROVIDER_IMAGE $$ISSUER $$PROVIDER_KEY_ARGS' < manifests/credential-provider.yaml.tmpl | $(KUBECTL) apply -f -
$(KUBECTL) -n agentmesh rollout status deployment --timeout=180s
$(KUBECTL) -n ate-system rollout status deployment/atenet-egress --timeout=120s
- @echo "mesh console: kubectl -n agentmesh port-forward svc/agentmesh-one 8070:8080 -> http://127.0.0.1:8070/console/"
@echo "next: make actor NAME=demo && make call NAME=demo"
+# Writes the provider's discovery document and JWKS for PROVIDER_KEY_PATH into
+# run/issuer/, to be published at ISSUER (for example
+# `gcloud storage cp -r run/issuer/* gs://<bucket>/substrate/`).
+issuer-docs:
+ @test -n "$(PROVIDER_KEY_PATH)" || { echo "PROVIDER_KEY_PATH is required"; exit 1; }
+ rm -rf run/issuer && mkdir -p run/issuer
+ cd credential-provider && go run . --print-discovery --print-discovery-dir ../run/issuer --issuer "$(ISSUER)" --jwks-uri "$(ISSUER)/keys" --signing-key-path "$(PROVIDER_KEY_PATH)"
+ @find run/issuer -type f
+
+run/join-token:
+ mkdir -p run && umask 077 && printf 'mesh_tok_%s' "$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" > $@
+
+run/admin-token:
+ mkdir -p run && umask 077 && printf 'mesh_adm_%s' "$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" > $@
+
+run/api-token:
+ mkdir -p run && umask 077 && head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n' > $@
+
# The actor's identity is spiffe://$(TRUST_DOMAIN)/actor/$(ATESPACE)/<name>,
# which Substrate derives from the certificate it gives the actor. The mesh
# policy in manifests/mesh.yaml.tmpl says what that identity may call.
@@ -87,8 +141,11 @@ actor:
@echo "actor $(NAME) is spiffe://$(TRUST_DOMAIN)/actor/$(ATESPACE)/$(NAME); try: make call NAME=$(NAME)"
call:
+ if (exec 3<>/dev/tcp/127.0.0.1/$(ROUTER_PORT)) 2>/dev/null; then
+ echo "127.0.0.1:$(ROUTER_PORT) is in use (a port-forward left from an earlier run?); free it or set ROUTER_PORT" >&2; exit 1
+ fi
$(KUBECTL) -n ate-system port-forward service/atenet-router $(ROUTER_PORT):80 >/dev/null 2>&1 &
- pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT; sleep 2
+ pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT INT TERM HUP; sleep 2
@echo "== tools the mesh offers actor $(NAME)"
ROUTER_PORT=$(ROUTER_PORT) ATESPACE=$(ATESPACE) python3 mcp.py tools $(NAME)
@echo "== mcp://calc, allowed for the actor role"
@@ -96,6 +153,28 @@ call:
@echo "== mcp://payroll, not in the actor role"
ROUTER_PORT=$(ROUTER_PORT) ATESPACE=$(ATESPACE) python3 mcp.py add $(NAME) payroll 2 3 || true
+# gemini calls Gemini on Vertex AI from inside the actor through the mesh (see
+# gemini.py). The policy must carry the egress destination and the http grant
+# (see the guide); GCP_PROJECT is the project of the destination; PEER is the
+# office node that serves it (default: this cluster's).
+GCP_PROJECT ?=
+GCP_REGION ?= us-central1
+GEMINI_MODEL ?= gemini-2.5-flash
+# AGENT_ENGINE=<reasoning engine id> adds a query to an agent on Agent Engine.
+AGENT_ENGINE ?=
+gemini:
+ @test -n "$(GCP_PROJECT)" || { echo "set GCP_PROJECT=<project that hosts the Vertex AI endpoint>"; exit 1; }
+ peer="$(PEER)"
+ if [ -z "$$peer" ]; then
+ peer=$$($(KUBECTL) -n agentmesh logs deploy/office-node | grep -o 'PeerID: 12D3Koo[A-Za-z0-9]*' | head -1 | cut -d' ' -f2)
+ fi
+ if (exec 3<>/dev/tcp/127.0.0.1/$(ROUTER_PORT)) 2>/dev/null; then
+ echo "127.0.0.1:$(ROUTER_PORT) is in use (a port-forward left from an earlier run?); free it or set ROUTER_PORT" >&2; exit 1
+ fi
+ $(KUBECTL) -n ate-system port-forward service/atenet-router $(ROUTER_PORT):80 >/dev/null 2>&1 &
+ pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT INT TERM HUP; sleep 2
+ ROUTER_PORT=$(ROUTER_PORT) ATESPACE=$(ATESPACE) python3 gemini.py $(NAME) $$peer $(GCP_PROJECT) $(GCP_REGION) $(GEMINI_MODEL) $(AGENT_ENGINE)
+
rogue:
$(MAKE) actor NAME=rogue
@echo "== rogue has an actor identity but the mesh policy binds it to no role"
@@ -105,10 +184,16 @@ rogue:
# Replaces the whole mesh policy. Nodes pick it up within 30s; an actor gets
# its new roles when the provider's cached credential expires (--cache-ttl).
policy:
- $(KUBECTL) -n agentmesh port-forward svc/agentmesh-one 18070:8080 >/dev/null 2>&1 &
- pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT; sleep 2
- curl -sS -X POST http://127.0.0.1:18070/policies \
- -H "Authorization: Bearer $$($(KUBECTL) -n agentmesh get secret agentmesh-tokens -o jsonpath='{.data.admin-token}' | base64 -d)" \
+ if [ "$(HUB)" = local ]; then
+ $(KUBECTL) -n agentmesh port-forward svc/agentmesh-one 18070:8080 >/dev/null 2>&1 &
+ pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT; sleep 2
+ hub=http://127.0.0.1:18070
+ else
+ hub=$(HUB)
+ fi
+ admin_token=$$(cat run/admin-token)
+ curl -sS -X POST "$$hub/policies" \
+ -H "Authorization: Bearer $$admin_token" \
-H 'Content-Type: application/json' \
--data-binary "$$($(KUBECTL) -n agentmesh get configmap mesh-config -o jsonpath='{.data.policy\.json}')"
@echo
@@ -120,7 +205,10 @@ status:
down:
-for a in demo rogue; do $(KATE) suspend actor $$a -a $(ATESPACE) >/dev/null 2>&1; $(KATE) delete actor $$a -a $(ATESPACE) >/dev/null 2>&1; done
- -$(KUBECTL) delete namespace agentmesh --wait=false
+ # The namespace and the credential-provider Service stay: the egress gateway
+ # was installed with that Service's address and holds its ClusterIP.
+ -$(KUBECTL) -n agentmesh delete deployment,configmap,secret,networkpolicy --all --timeout=120s
+ -$(KUBECTL) -n agentmesh delete service edge-node calc-mcp --ignore-not-found
clean:
kind delete cluster --name $(KIND_CLUSTER_NAME)
diff --git a/development/examples/substrate-actor/README.md b/development/examples/substrate-actor/README.md
index 93ddb0b9..3c870702 100644
--- a/development/examples/substrate-actor/README.md
+++ b/development/examples/substrate-actor/README.md
@@ -1,19 +1,20 @@
# A Substrate actor on the mesh
-Everything the guide at `site/content/docs/guides/substrate.md` walks through,
-on a kind cluster:
+This example runs the full procedure of the guide at
+`site/content/docs/guides/substrate.md` on a kind cluster:
-- Agent Substrate installed from a checkout, with its egress gateway pointed
- at the Agent Mesh credential provider;
-- `credential-provider/`, a Substrate `CredentialProvider` that answers
+- Agent Substrate, installed from a checkout, with its egress gateway pointed
+ at the Agent Mesh credential provider.
+- `credential-provider/`, a Substrate `CredentialProvider`. It answers
`ate-secret://agentmesh/mesh/biscuit` with a mesh credential bound to the
- calling actor's SPIFFE ID, by asserting that identity to the mesh as a
- workload identity issuer;
-- `agentmesh-one`, an edge `agentmesh-node` the actors reach, and an office
- node publishing a calculator as `mcp://calc` and `mcp://payroll`, all in
- namespace `agentmesh`;
-- an `EgressPolicy` that lets the actor reach the edge node over plain HTTP and
- has the gateway inject the credential as `X-Mesh-Authentication`.
+ SPIFFE ID of the calling actor. It asserts that identity to the mesh as a
+ workload identity issuer.
+- `agentmesh-one`, an edge `agentmesh-node` that the actors reach, and an
+ office node that publishes a calculator as `mcp://calc` and
+ `mcp://payroll`. All run in namespace `agentmesh`.
+- An `EgressPolicy` that lets the actor reach the edge node over plain HTTP.
+ The `EgressPolicy` makes the egress gateway inject the credential as
+ `X-Mesh-Authentication`.
```bash
make cluster SUBSTRATE_DIR=~/src/substrate # kind + Substrate (builds images; a few minutes)
@@ -21,11 +22,63 @@ make up # the mesh and the provider
make actor NAME=demo # a sandbox actor with the egress policy
make call NAME=demo # tools, calc allowed, payroll refused
make rogue # an actor the policy does not bind: refused
+make gemini NAME=demo GCP_PROJECT=<project> # Gemini on Vertex AI through the mesh; see the guide
make policy # push the policy after editing manifests/mesh.yaml.tmpl
make down # the actors and the agentmesh namespace
make clean # the kind cluster
```
-The provider has its own Go module against the Substrate release it builds
-against (`credential-provider/go.mod`); `go test ./...` in that directory runs
-its unit tests without a cluster.
+The provider has its own Go module (`credential-provider/go.mod`) that
+depends on the Substrate release it builds against. `go test ./...` in that
+directory runs its unit tests without a cluster.
+
+## With the hub elsewhere (Cloud Run)
+
+`HUB=<url>` joins an `agentmesh-one` that runs outside the cluster. The
+example then does not deploy the in-cluster hub. The control plane must
+fetch the discovery document and the JWKS of the provider. A kind cluster
+cannot serve them publicly. Therefore the provider signs with a key that you
+keep. You publish the two documents as static files at the location that
+`ISSUER` names, a public bucket:
+
+```bash
+openssl genpkey -algorithm ec -pkeyopt ec_paramgen_curve:P-256 -out provider-p256.pem
+export HUB=https://<hub>.run.app ISSUER=https://storage.googleapis.com/<bucket>/substrate PROVIDER_KEY_PATH=$PWD/provider-p256.pem
+make issuer-docs && gcloud storage cp -r run/issuer/.well-known run/issuer/keys gs://<bucket>/substrate/
+# hub: --workload-issuer=<other issuers>,$ISSUER (on Cloud Run pass the list with --args="^;^...")
+cp <hub join token> run/join-token; cp <hub admin token> run/admin-token
+make up && make actor NAME=demo && make call NAME=demo
+```
+
+The provider mounts the key from the Secret `credential-provider-signing-key`.
+Its discovery document (`--jwks-uri`) names the JWKS in the bucket. To
+rotate the key, generate a new key. Publish a JWKS that carries both keys.
+Switch the Secret. Then remove the old key from the JWKS.
+
+
+## On GKE
+
+The Substrate tooling creates the cluster and installs Substrate. This
+example adds the mesh to that cluster. `KUBE_CONTEXT` selects the cluster.
+`CALC_MCP_IMAGE` names an image that the nodes can pull, because there is no
+kind cluster to load an image into:
+
+```bash
+# in the Substrate checkout: cp hack/ate-dev-env.sh.example .ate-dev-env.sh, edit, then
+source .ate-dev-env.sh && go run ./tools/setup-gcp bootstrap # cluster, bucket, IAM (~8 min)
+kubectl apply -f <this dir>/manifests/namespace.yaml # the provider's Service must resolve before the gateway starts
+./hack/install-ate.sh --deploy-ate-system --deploy-demo-sandbox \
+ --credential-provider='{"name":"agentmesh","address":"credential-provider.agentmesh.svc:50051"}'
+# back here
+docker build -t <repo>/calc-mcp:latest ../calc-mcp && docker push <repo>/calc-mcp:latest
+export KUBE_CONTEXT=gke_<project>_<zone>_<cluster> KO_DOCKER_REPO=<repo> CALC_MCP_IMAGE=<repo>/calc-mcp:latest
+export HUB=... ISSUER=... PROVIDER_KEY_PATH=... # as in the section above
+make up && make actor NAME=demo && make call NAME=demo
+```
+
+`atenet-egress` stays at `1/3` until `make up` creates the provider that it
+was installed to call. Two Substrate clusters on one hub present the same
+actor identities. The trust domain in
+`spiffe://substrate-actor.local/actor/<atespace>/<name>` belongs to
+Substrate, not to the cluster. Run one cluster per hub, or bind by atespace
+names that differ per cluster.
diff --git a/development/examples/substrate-actor/credential-provider/main.go b/development/examples/substrate-actor/credential-provider/main.go
index 327b7d56..a491c9af 100644
--- a/development/examples/substrate-actor/credential-provider/main.go
+++ b/development/examples/substrate-actor/credential-provider/main.go
@@ -47,6 +47,7 @@ import (
"net/url"
"os"
"os/signal"
+ "path/filepath"
"strings"
"sync"
"syscall"
@@ -82,6 +83,10 @@ var (
issuer = flag.String("issuer", "", "iss of the assertions this provider signs; the control plane fetches <issuer>/.well-known/openid-configuration (required)")
audience = flag.String("audience", "agentmesh", "aud of the assertions; one of the control plane's --allowed-audiences")
cacheTTL = flag.Duration("cache-ttl", time.Minute, "longest a minted credential is reused; also how long a mesh policy change takes to reach an actor")
+ signingKeyPath = flag.String("signing-key-path", "", "PEM file (PKCS#8 or SEC 1) with the P-256 key the assertions are signed with; generated per run when empty, in which case the issuer must be this provider's own HTTP address")
+ jwksURI = flag.String("jwks-uri", "", "jwks_uri the discovery document names; defaults to <issuer>/keys. Set it when the documents are published elsewhere (a bucket) and the issuer is not this process")
+ printDiscovery = flag.Bool("print-discovery", false, "write the discovery document and the JWKS for --signing-key-path to --print-discovery-dir and exit, for publishing them where the control plane reads the issuer")
+ printDiscoveryTo = flag.String("print-discovery-dir", ".", "directory --print-discovery writes .well-known/openid-configuration and keys into")
)
func main() {
@@ -95,6 +100,16 @@ func main() {
}
func run(ctx context.Context) error {
+ if *printDiscovery {
+ if *issuer == "" || *signingKeyPath == "" {
+ return errors.New("--print-discovery needs --issuer and --signing-key-path")
+ }
+ signer, err := newSigner(*issuer, *audience, *signingKeyPath, *jwksURI)
+ if err != nil {
+ return err
+ }
+ return signer.writeDiscovery(*printDiscoveryTo)
+ }
for name, value := range map[string]string{
"--server-cred-bundle": *serverBundle, "--client-ca-file": *clientCAFile,
"--node-url": *nodeURL, "--issuer": *issuer,
@@ -106,7 +121,7 @@ func run(ctx context.Context) error {
ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM)
defer stop()
- signer, err := newSigner(*issuer, *audience)
+ signer, err := newSigner(*issuer, *audience, *signingKeyPath, *jwksURI)
if err != nil {
return err
}
@@ -132,7 +147,7 @@ func run(ctx context.Context) error {
return fmt.Errorf("listen %s: %w", *listenAddr, err)
}
- httpSrv := &http.Server{Addr: *httpAddr, Handler: signer.handler(), ReadHeaderTimeout: 5 * time.Second}
+ httpSrv := &http.Server{Addr: *httpAddr, Handler: signer.handler(provider.cacheSize), ReadHeaderTimeout: 5 * time.Second}
errs := make(chan error, 2)
go func() { errs <- grpcSrv.Serve(lis) }()
go func() { errs <- httpSrv.ListenAndServe() }()
@@ -180,19 +195,32 @@ type actorRef struct{ atespace, name string }
// is not served.
func (p *provider) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecretRequest) (*credproviderpb.FetchSecretResponse, error) {
if err := p.checkURI(req.GetUri()); err != nil {
+ slog.Warn("refused: not this provider's credential", slog.String("actor", req.GetActorSpiffeId()), slog.String("uri", req.GetUri()), slog.Any("err", err))
return nil, status.Error(codes.NotFound, err.Error())
}
actor, err := p.parseActor(req.GetActorSpiffeId())
if err != nil {
+ slog.Warn("refused: not an actor of this trust domain", slog.String("actor", req.GetActorSpiffeId()), slog.Any("err", err))
return nil, status.Error(codes.PermissionDenied, err.Error())
}
token, err := p.credential(ctx, req.GetActorSpiffeId(), actor)
if err != nil {
+ // mint logs a refusal by the mesh with its status; this is every other
+ // way the credential did not come (the node unreachable, the signer).
+ slog.Warn("no credential for the actor", slog.String("actor", req.GetActorSpiffeId()), slog.Any("err", err))
return nil, err
}
return &credproviderpb.FetchSecretResponse{OpaqueBytes: token}, nil
}
+// cacheSize is the number of actors with a credential entry, read from the
+// live map for the gauge on /metrics.
+func (p *provider) cacheSize() int {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return len(p.cache)
+}
+
func (p *provider) uri() string {
return "ate-secret://" + p.name + "/" + p.instance + "/" + credentialTail
}
@@ -305,10 +333,27 @@ type signer struct {
jwks, discovery []byte
}
-func newSigner(issuer, audience string) (*signer, error) {
- key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
- if err != nil {
- return nil, err
+// newSigner signs with the P-256 key at keyPath, or with one generated for
+// this run when keyPath is empty. A persistent key is what lets the
+// discovery document and the JWKS be published as static files somewhere
+// the control plane can read (a bucket), with the issuer naming that place
+// and jwksURI the JWKS there; an ephemeral key needs the issuer to be this
+// process, since the keys change with every start.
+func newSigner(issuer, audience, keyPath, jwksURI string) (*signer, error) {
+ var key *ecdsa.PrivateKey
+ if keyPath == "" {
+ var err error
+ if key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader); err != nil {
+ return nil, err
+ }
+ } else {
+ var err error
+ if key, err = loadP256Key(keyPath); err != nil {
+ return nil, err
+ }
+ }
+ if jwksURI == "" {
+ jwksURI = issuer + "/keys"
}
der, err := x509.MarshalPKIXPublicKey(&key.PublicKey)
if err != nil {
@@ -327,7 +372,7 @@ func newSigner(issuer, audience string) (*signer, error) {
}}})
s.discovery, _ = json.Marshal(map[string]any{
"issuer": issuer,
- "jwks_uri": issuer + "/keys",
+ "jwks_uri": jwksURI,
"response_types_supported": []string{"id_token"},
"subject_types_supported": []string{"public"},
"id_token_signing_alg_values_supported": []string{"ES256"},
@@ -336,6 +381,48 @@ func newSigner(issuer, audience string) (*signer, error) {
return s, nil
}
+// loadP256Key reads one PEM private key, PKCS#8 or SEC 1, and requires P-256.
+func loadP256Key(path string) (*ecdsa.PrivateKey, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, fmt.Errorf("read signing key: %w", err)
+ }
+ block, _ := pem.Decode(data)
+ if block == nil {
+ return nil, fmt.Errorf("signing key %s: no PEM block", path)
+ }
+ var parsed any
+ switch block.Type {
+ case "PRIVATE KEY":
+ parsed, err = x509.ParsePKCS8PrivateKey(block.Bytes)
+ case "EC PRIVATE KEY":
+ parsed, err = x509.ParseECPrivateKey(block.Bytes)
+ default:
+ return nil, fmt.Errorf("signing key %s: unsupported PEM block %q", path, block.Type)
+ }
+ if err != nil {
+ return nil, fmt.Errorf("signing key %s: %w", path, err)
+ }
+ key, ok := parsed.(*ecdsa.PrivateKey)
+ if !ok || key.Curve != elliptic.P256() {
+ return nil, fmt.Errorf("signing key %s: want a P-256 key, got %T", path, parsed)
+ }
+ return key, nil
+}
+
+// writeDiscovery writes the two documents an OIDC relying party fetches,
+// at the paths they are fetched from, so a bucket mirrors the issuer.
+func (s *signer) writeDiscovery(dir string) error {
+ wk := filepath.Join(dir, ".well-known")
+ if err := os.MkdirAll(wk, 0o755); err != nil {
+ return err
+ }
+ if err := os.WriteFile(filepath.Join(wk, "openid-configuration"), s.discovery, 0o644); err != nil {
+ return err
+ }
+ return os.WriteFile(filepath.Join(dir, "keys"), s.jwks, 0o644)
+}
+
func (s *signer) assertion(subject string, actor map[string]string) (string, error) {
now := time.Now()
jti := make([]byte, 16)
@@ -356,7 +443,10 @@ func (s *signer) assertion(subject string, actor map[string]string) (string, err
return token.SignedString(s.key)
}
-func (s *signer) handler() http.Handler {
+// handler serves the issuer documents, the probes, and /metrics with the
+// one piece of state this process holds: the actors it has a credential
+// cached for, read from the live map.
+func (s *signer) handler(cacheSize func() int) http.Handler {
mux := http.NewServeMux()
serve := func(body []byte) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
@@ -369,6 +459,12 @@ func (s *signer) handler() http.Handler {
mux.Handle("GET /keys", serve(s.jwks))
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })
mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })
+ mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/plain; version=0.0.4")
+ fmt.Fprintf(w, "# HELP agentmesh_credential_provider_cached_actors Actors this provider holds a mesh credential for\n")
+ fmt.Fprintf(w, "# TYPE agentmesh_credential_provider_cached_actors gauge\n")
+ fmt.Fprintf(w, "agentmesh_credential_provider_cached_actors %d\n", cacheSize())
+ })
return mux
}
diff --git a/development/examples/substrate-actor/credential-provider/main_test.go b/development/examples/substrate-actor/credential-provider/main_test.go
index 338fbd59..c603b69f 100644
--- a/development/examples/substrate-actor/credential-provider/main_test.go
+++ b/development/examples/substrate-actor/credential-provider/main_test.go
@@ -16,9 +16,19 @@ package main
import (
"context"
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "crypto/x509"
"encoding/json"
+ "encoding/pem"
+ "fmt"
+ "io"
"net/http"
"net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
"sync/atomic"
"testing"
"time"
@@ -71,7 +81,7 @@ func fakeNode(t *testing.T, s *signer, bound string, calls *atomic.Int32) *httpt
func newTestProvider(t *testing.T, node string, ttl time.Duration) (*provider, *signer) {
t.Helper()
- s, err := newSigner("http://provider.test", "agentmesh")
+ s, err := newSigner("http://provider.test", "agentmesh", "", "")
if err != nil {
t.Fatal(err)
}
@@ -162,11 +172,11 @@ func TestCacheHonoursTTL(t *testing.T) {
}
func TestDiscoveryDocument(t *testing.T) {
- s, err := newSigner("http://provider.test", "agentmesh")
+ s, err := newSigner("http://provider.test", "agentmesh", "", "")
if err != nil {
t.Fatal(err)
}
- srv := httptest.NewServer(s.handler())
+ srv := httptest.NewServer(s.handler(func() int { return 0 }))
defer srv.Close()
resp, err := http.Get(srv.URL + "/.well-known/openid-configuration")
if err != nil {
@@ -196,3 +206,119 @@ func TestDiscoveryDocument(t *testing.T) {
t.Fatalf("jwks: %v %+v", err, set)
}
}
+
+// TestSignerPersistentKeyAndStaticDiscovery is the issuer-on-a-bucket case:
+// the same key file yields the same JWKS across runs, the discovery document
+// names the bucket's JWKS, and an assertion signed in one run verifies with
+// the JWKS written by another.
+func TestSignerPersistentKeyAndStaticDiscovery(t *testing.T) {
+ key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ der, err := x509.MarshalPKCS8PrivateKey(key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ keyPath := filepath.Join(t.TempDir(), "p256.pem")
+ if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ const issuer = "https://storage.googleapis.com/bucket/issuer"
+ const jwksURI = issuer + "/keys"
+
+ first, err := newSigner(issuer, "agentmesh", keyPath, jwksURI)
+ if err != nil {
+ t.Fatalf("newSigner: %v", err)
+ }
+ second, err := newSigner(issuer, "agentmesh", keyPath, jwksURI)
+ if err != nil {
+ t.Fatalf("newSigner (second run): %v", err)
+ }
+ if string(first.jwks) != string(second.jwks) || first.kid != second.kid {
+ t.Fatal("two runs on the same key file produced different JWKS; the published documents would not match the signer")
+ }
+ dir := t.TempDir()
+ if err := first.writeDiscovery(dir); err != nil {
+ t.Fatalf("writeDiscovery: %v", err)
+ }
+ disc, err := os.ReadFile(filepath.Join(dir, ".well-known", "openid-configuration"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var doc map[string]any
+ if err := json.Unmarshal(disc, &doc); err != nil {
+ t.Fatal(err)
+ }
+ if doc["issuer"] != issuer || doc["jwks_uri"] != jwksURI {
+ t.Fatalf("discovery = %v, want issuer %s and jwks_uri %s", doc, issuer, jwksURI)
+ }
+ jwksFile, err := os.ReadFile(filepath.Join(dir, "keys"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(jwksFile) != string(second.jwks) {
+ t.Fatal("the JWKS written for publishing differs from the one a later run would sign under")
+ }
+
+ assertion, err := second.assertion("spiffe://substrate-actor.local/actor/a/b", nil)
+ if err != nil {
+ t.Fatalf("assertion: %v", err)
+ }
+ parsed, err := jwt.Parse(assertion, func(tok *jwt.Token) (any, error) {
+ if kid, _ := tok.Header["kid"].(string); kid != first.kid {
+ return nil, fmt.Errorf("kid %q not in the published JWKS", kid)
+ }
+ return &key.PublicKey, nil
+ }, jwt.WithValidMethods([]string{"ES256"}))
+ if err != nil || !parsed.Valid {
+ t.Fatalf("assertion does not verify against the published key: %v", err)
+ }
+
+ if _, err := newSigner(issuer, "agentmesh", filepath.Join(t.TempDir(), "absent.pem"), ""); err == nil {
+ t.Fatal("newSigner with a missing key file returned nil")
+ }
+}
+
+// /metrics reports the actors with a cached credential from the live map:
+// one after a mint, still one after a second request for the same actor,
+// two after a request for another actor.
+func TestMetricsReportLiveCacheSize(t *testing.T) {
+ s, err := newSigner("http://provider.test", "agentmesh", "", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ p := &provider{name: "agentmesh", instance: "mesh", trustDomain: "substrate-actor.local", signer: s,
+ cacheTTL: time.Minute, cache: map[string]*cached{}}
+ srv := httptest.NewServer(s.handler(p.cacheSize))
+ defer srv.Close()
+ read := func(t *testing.T) string {
+ t.Helper()
+ resp, err := http.Get(srv.URL + "/metrics")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ body, _ := io.ReadAll(resp.Body)
+ for _, line := range strings.Split(string(body), "\n") {
+ if strings.HasPrefix(line, "agentmesh_credential_provider_cached_actors ") {
+ return strings.TrimPrefix(line, "agentmesh_credential_provider_cached_actors ")
+ }
+ }
+ t.Fatalf("no gauge line in %q", body)
+ return ""
+ }
+ if got := read(t); got != "0" {
+ t.Fatalf("empty cache: gauge %s, want 0", got)
+ }
+ for _, actor := range []string{"spiffe://substrate-actor.local/actor/a/one", "spiffe://substrate-actor.local/actor/a/one", "spiffe://substrate-actor.local/actor/a/two"} {
+ p.mu.Lock()
+ if _, ok := p.cache[actor]; !ok {
+ p.cache[actor] = &cached{token: []byte("t"), expires: time.Now().Add(time.Minute)}
+ }
+ p.mu.Unlock()
+ }
+ if got := read(t); got != "2" {
+ t.Fatalf("two actors cached: gauge %s, want 2", got)
+ }
+}
diff --git a/development/examples/substrate-actor/gemini.py b/development/examples/substrate-actor/gemini.py
new file mode 100755
index 00000000..85ff3ea3
--- /dev/null
+++ b/development/examples/substrate-actor/gemini.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# Calls Gemini on Vertex AI from inside a Substrate actor through the mesh.
+# The request goes to the edge node, which forwards it to the office node that
+# serves egress://aiplatform.googleapis.com; that node authorizes the method
+# and path, mints a federated Google token and injects it. The actor holds no
+# Google credential, only the placeholder the egress gateway replaces with its
+# mesh credential.
+#
+# gemini.py ACTOR OFFICE_PEER PROJECT [REGION] [MODEL] [AGENT_ENGINE_ID]
+#
+# With an Agent Engine id, a fourth request queries that agent: the same host
+# under another granted path.
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+
+ROUTER = "http://127.0.0.1:" + os.environ.get("ROUTER_PORT", "8000")
+ATESPACE = os.environ.get("ATESPACE", "ate-demo-sandbox")
+# Absolute name: see mcp.py.
+EDGE = "http://edge-node.agentmesh.svc.cluster.local.:8080"
+HEADERS = ["--header=Content-Type: application/json", "--header=X-Mesh-Authentication: ******"]
+
+
+def in_actor(actor, command, timeout="60s"):
+ body = json.dumps({"command": command, "timeout": timeout}).encode()
+ req = urllib.request.Request(ROUTER + "/process", data=body, method="POST", headers={
+ "Content-Type": "application/json",
+ "ate-target-actor": f"{ATESPACE}/{actor}",
+ })
+ try:
+ with urllib.request.urlopen(req, timeout=120) as resp:
+ out = json.load(resp)
+ except urllib.error.HTTPError as e:
+ sys.exit(f"router answered HTTP {e.code} for actor {ATESPACE}/{actor}: {e.read().decode()[:300]}")
+ # A non-zero exit (wget on a 4xx) comes back as an error with the stderr
+ # attached; the status line is what the caller wants from it.
+ if out.get("error") and "HTTP/" not in out.get("stderr", ""):
+ sys.exit(f"actor could not run the command: {out['error']} {out.get('stderr', '')}")
+ return out.get("stdout", ""), out.get("stderr", "")
+
+
+def status_line(stderr):
+ # wget -S writes the status line to stderr.
+ for line in stderr.splitlines():
+ if line.strip().startswith("HTTP/"):
+ return line.strip()
+ return "(no status line)"
+
+
+def main():
+ if len(sys.argv) < 4:
+ sys.exit(__doc__)
+ actor, peer, project = sys.argv[1:4]
+ region = sys.argv[4] if len(sys.argv) > 4 else "us-central1"
+ model = sys.argv[5] if len(sys.argv) > 5 else "gemini-2.5-flash"
+ engine = sys.argv[6] if len(sys.argv) > 6 else ""
+ base = f"{EDGE}/mesh/{peer}/egress/aiplatform.googleapis.com/v1/projects/{project}/locations/{region}"
+
+ print(f"== POST {model}:generateContent, granted to the actor role")
+ body = json.dumps({"contents": [{"role": "user", "parts": [{"text": "Answer with one word: what is 2+3?"}]}]})
+ out, err = in_actor(actor, ["wget", "-q", "-O", "-", "-T", "60", *HEADERS, f"--post-data={body}",
+ f"{base}/publishers/google/models/{model}:generateContent"])
+ try:
+ print(" answer:", json.loads(out)["candidates"][0]["content"]["parts"][0]["text"].strip())
+ except (ValueError, KeyError, IndexError):
+ print(" refused:", (out or err).strip()[:300])
+
+ print("== GET on the model, a method the grant does not list")
+ _, err = in_actor(actor, ["wget", "-S", "-q", "-O", "/dev/null", "-T", "30", *HEADERS,
+ f"{base}/publishers/google/models/{model}"])
+ print(" ", status_line(err))
+
+ print("== POST to another Vertex AI path, outside the granted paths")
+ _, err = in_actor(actor, ["wget", "-S", "-q", "-O", "/dev/null", "-T", "30", *HEADERS, "--post-data={}",
+ f"{base}/endpoints"])
+ print(" ", status_line(err))
+
+ if engine:
+ print(f"== POST reasoningEngines/{engine}:query, an agent on Agent Engine")
+ body = json.dumps({"input": {"a": 40, "b": 2}})
+ out, err = in_actor(actor, ["wget", "-q", "-O", "-", "-T", "60", *HEADERS, f"--post-data={body}",
+ f"{base}/reasoningEngines/{engine}:query"])
+ try:
+ print(" output:", json.dumps(json.loads(out).get("output")))
+ except ValueError:
+ print(" refused:", (out or err).strip()[:300])
+
+
+if __name__ == "__main__":
+ main()
diff --git a/development/examples/substrate-actor/manifests/credential-provider.yaml.tmpl b/development/examples/substrate-actor/manifests/credential-provider.yaml.tmpl
index 56590d15..720276fa 100644
--- a/development/examples/substrate-actor/manifests/credential-provider.yaml.tmpl
+++ b/development/examples/substrate-actor/manifests/credential-provider.yaml.tmpl
@@ -29,7 +29,8 @@ spec:
- --provider-name=agentmesh
- --node-url=http://edge-node.agentmesh.svc.cluster.local:8080
# Must equal the control plane's --workload-issuer, verbatim.
- - --issuer=http://credential-provider.agentmesh.svc.cluster.local:8080
+ - --issuer=${ISSUER}
+ ${PROVIDER_KEY_ARGS}
- --audience=agentmesh
- --cache-ttl=60s
ports:
@@ -40,7 +41,12 @@ spec:
volumeMounts:
- {name: servicedns, mountPath: /run/servicedns.podcert.ate.dev, readOnly: true}
- {name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true}
+ - {name: signing-key, mountPath: /run/agentmesh-provider, readOnly: true}
volumes:
+ - name: signing-key
+ secret:
+ secretName: credential-provider-signing-key
+ optional: true
- name: servicedns
projected:
sources:
diff --git a/development/examples/substrate-actor/manifests/hub.yaml.tmpl b/development/examples/substrate-actor/manifests/hub.yaml.tmpl
new file mode 100644
index 00000000..dc32c918
--- /dev/null
+++ b/development/examples/substrate-actor/manifests/hub.yaml.tmpl
@@ -0,0 +1,53 @@
+# agentmesh-one in the cluster, used with HUB=local. With HUB=<url> the
+# nodes and the provider use an agentmesh-one elsewhere and this is not applied.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: agentmesh-one
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: agentmesh-one}
+ template:
+ metadata:
+ labels: {app: agentmesh-one}
+ spec:
+ containers:
+ - name: agentmesh-one
+ image: ${AGENTMESH_ONE_IMAGE}
+ args:
+ - --bind-address=0.0.0.0
+ - --port=8080
+ - --data-dir=/data
+ - --policy-file=/etc/agentmesh/policy.json
+ - --token-path=/run/agentmesh/join-token
+ - --admin-token-path=/run/agentmesh/admin-token
+ # The credential provider asserts actor identities; the mesh trusts
+ # it as a workload identity issuer and binds roles to the subject.
+ - --workload-issuer=${ISSUER}
+ - --allowed-audiences=agentmesh
+ - --enroll-qr=false
+ ports:
+ - {name: http, containerPort: 8080}
+ volumeMounts:
+ - {name: config, mountPath: /etc/agentmesh, readOnly: true}
+ - {name: tokens, mountPath: /run/agentmesh, readOnly: true}
+ - {name: data, mountPath: /data}
+ volumes:
+ - name: config
+ configMap: {name: mesh-config}
+ - name: tokens
+ secret: {secretName: agentmesh-tokens, defaultMode: 0444}
+ - name: data
+ emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: agentmesh-one
+ namespace: agentmesh
+spec:
+ selector: {app: agentmesh-one}
+ ports:
+ - {name: http, port: 8080, targetPort: http}
diff --git a/development/examples/substrate-actor/manifests/mesh.yaml.tmpl b/development/examples/substrate-actor/manifests/mesh.yaml.tmpl
index 584e1628..c87d564b 100644
--- a/development/examples/substrate-actor/manifests/mesh.yaml.tmpl
+++ b/development/examples/substrate-actor/manifests/mesh.yaml.tmpl
@@ -15,7 +15,7 @@ data:
policy.json: |
{
"roles": [
- {"name": "mesh:role:node", "allowed_services": ["*"], "allowed_targets": ["*"], "allowed_labels": ["*"]},
+ {"name": "mesh:role:node", "allowed_services": ["*"], "allowed_publish": ["*"], "allowed_targets": ["*"], "allowed_labels": ["*"]},
{"name": "mesh:role:router", "allowed_services": ["*"], "allowed_targets": ["*"]},
{"name": "actor",
"allowed_services": ["system://mesh.catalog", "mcp://calc"],
@@ -41,58 +41,6 @@ data:
---
apiVersion: apps/v1
kind: Deployment
-metadata:
- name: agentmesh-one
- namespace: agentmesh
-spec:
- replicas: 1
- selector:
- matchLabels: {app: agentmesh-one}
- template:
- metadata:
- labels: {app: agentmesh-one}
- spec:
- containers:
- - name: agentmesh-one
- image: ${AGENTMESH_ONE_IMAGE}
- args:
- - --bind-address=0.0.0.0
- - --port=8080
- - --data-dir=/data
- - --policy-file=/etc/agentmesh/policy.json
- - --token-path=/run/agentmesh/join-token
- - --admin-token-path=/run/agentmesh/admin-token
- # The credential provider asserts actor identities; the mesh trusts
- # it as a workload identity issuer and binds roles to the subject.
- - --workload-issuer=http://credential-provider.agentmesh.svc.cluster.local:8080
- - --allowed-audiences=agentmesh
- - --enroll-qr=false
- ports:
- - {name: http, containerPort: 8080}
- volumeMounts:
- - {name: config, mountPath: /etc/agentmesh, readOnly: true}
- - {name: tokens, mountPath: /run/agentmesh, readOnly: true}
- - {name: data, mountPath: /data}
- volumes:
- - name: config
- configMap: {name: mesh-config}
- - name: tokens
- secret: {secretName: agentmesh-tokens, defaultMode: 0444}
- - name: data
- emptyDir: {}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: agentmesh-one
- namespace: agentmesh
-spec:
- selector: {app: agentmesh-one}
- ports:
- - {name: http, port: 8080, targetPort: http}
----
-apiVersion: apps/v1
-kind: Deployment
metadata:
name: edge-node
namespace: agentmesh
@@ -109,15 +57,19 @@ spec:
image: ${AGENTMESH_NODE_IMAGE}
args:
- run
- - --control-plane=http://agentmesh-one.agentmesh.svc.cluster.local:8080
- - --insecure-control-plane
+ - --control-plane=${CONTROL_PLANE_URL}
+ ${CONTROL_PLANE_INSECURE_ARG}
- --bootstrap-token-path=/run/agentmesh/join-token
- --api-token-path=/run/agentmesh/api-token
- --data-dir=/data
- --bind-addr=0.0.0.0:8080
- --control-plane-sync-interval=30s
+ - --metrics-addr=0.0.0.0:9090
ports:
- {name: http, containerPort: 8080}
+ readinessProbe:
+ httpGet: {path: /readyz, port: 9090}
+ periodSeconds: 2
volumeMounts:
- {name: tokens, mountPath: /run/agentmesh, readOnly: true}
- {name: data, mountPath: /data}
@@ -155,14 +107,18 @@ spec:
image: ${AGENTMESH_NODE_IMAGE}
args:
- run
- - --control-plane=http://agentmesh-one.agentmesh.svc.cluster.local:8080
- - --insecure-control-plane
+ - --control-plane=${CONTROL_PLANE_URL}
+ ${CONTROL_PLANE_INSECURE_ARG}
- --bootstrap-token-path=/run/agentmesh/join-token
- --api-token-path=/run/agentmesh/api-token
- --data-dir=/data
- --bind-addr=
- --config=/etc/agentmesh/office.yaml
- --control-plane-sync-interval=30s
+ - --metrics-addr=0.0.0.0:9090
+ readinessProbe:
+ httpGet: {path: /readyz, port: 9090}
+ periodSeconds: 2
volumeMounts:
- {name: config, mountPath: /etc/agentmesh, readOnly: true}
- {name: tokens, mountPath: /run/agentmesh, readOnly: true}
@@ -203,3 +159,4 @@ spec:
selector: {app: calc-mcp}
ports:
- {name: http, port: 7777, targetPort: http}
+
diff --git a/development/examples/substrate-actor/mcp.py b/development/examples/substrate-actor/mcp.py
index be615c0d..3c099834 100755
--- a/development/examples/substrate-actor/mcp.py
+++ b/development/examples/substrate-actor/mcp.py
@@ -17,7 +17,9 @@ import urllib.request
ROUTER = "http://127.0.0.1:" + os.environ.get("ROUTER_PORT", "8000")
ATESPACE = os.environ.get("ATESPACE", "ate-demo-sandbox")
-NODE = "http://edge-node.agentmesh.svc.cluster.local:8080/mcp"
+# Absolute name: the actor's DNS relay answers REFUSED for the search-domain
+# expansions musl tries first under ndots:5, and musl treats that as failure.
+NODE = "http://edge-node.agentmesh.svc.cluster.local.:8080/mcp"
def in_actor(actor, command, timeout="60s"):
diff --git a/site/content/docs/guides/openshell-and-substrate.md b/site/content/docs/guides/openshell-and-substrate.md
new file mode 100644
index 00000000..eb88228b
--- /dev/null
+++ b/site/content/docs/guides/openshell-and-substrate.md
@@ -0,0 +1,637 @@
+---
+title: "OpenShell and Substrate"
+linkTitle: "OpenShell and Substrate"
+weight: 8
+description: "Put NVIDIA OpenShell sandboxes and Agent Substrate actors on one mesh, on Kubernetes you manage or on GKE, and move a workload between the two runtimes with a policy change."
+---
+
+[NVIDIA OpenShell](https://github.com/NVIDIA/OpenShell) and
+[Agent Substrate](https://github.com/agent-substrate/substrate) run agents in
+isolated workloads. OpenShell runs an agent in a sandbox. Substrate runs an
+agent in an actor. Both runtimes keep the credentials and the egress rules of
+the agent outside the workload.
+
+This tutorial puts workloads of both runtimes on one Agent Mesh. Each
+workload is a mesh member with its own identity. One policy says what each
+identity can call. A workload in one cluster can reach a tool that a node
+publishes in the other cluster. To move a workload to the other runtime, run
+the same image there and change one line of the policy.
+
+The tutorial covers four deployments and the paths between them:
+
+| | Kubernetes you manage | GKE |
+|---|---|---|
+| **OpenShell** | [Section 3](#3-openshell-on-kubernetes-you-manage) | [Section 4](#4-openshell-on-gke) |
+| **Substrate** | [Section 5](#5-substrate-on-kubernetes-you-manage) | [Section 6](#6-substrate-on-gke) |
+
+kind represents Kubernetes that you manage. Every command in this tutorial
+ran on the versions in [Section 1.3](#13-versions).
+
+## 1. How it works
+
+### 1.1 Three products, three meanings of "gateway"
+
+| Role | OpenShell | Substrate | Agent Mesh |
+|---|---|---|---|
+| Control plane: API, lifecycle, policy, credentials | **gateway** (`ghcr.io/nvidia/openshell/gateway`). It carries no agent traffic. | `ateapi` and `atecontroller` | `agentmesh-one` (control plane and router) |
+| Per-workload enforcement point: DNS, egress rules, credential injection | **supervisor**, one per sandbox | `atunnel` in the worker pod and the shared `atenet-egress` gateway | `agentmesh-node` |
+| Workload | sandbox, with networking off | actor in gVisor or a micro-VM | any client of a node |
+
+Agent Mesh connects to the enforcement point of each runtime. The allowlist
+of the runtime contains one external endpoint, the edge `agentmesh-node`. The
+mesh decides which tools, models and destinations the workload can reach, and
+knows the identity under which the workload calls them. The data path of the
+runtime does not change.
+
+### 1.2 How a sandbox becomes a mesh member
+
+```mermaid
+sequenceDiagram
+ participant R as Runtime enforcement point (OpenShell supervisor or Substrate egress gateway)
+ participant N as edge agentmesh-node
+ participant CP as agentmesh-one
+ participant S as office agentmesh-node
+ R->>N: POST /oauth/token (JWT that proves the sandbox's SPIFFE ID)
+ N->>CP: POST /token/exchange (signed by the node)
+ Note over CP: verifies the JWT against --workload-issuer,<br/>checks aud is in --allowed-audiences,<br/>roles = bindings that match user:<sub>
+ CP-->>N: Biscuit (subject, actor_node, roles, expire_time)
+ N-->>R: access_token
+ R->>N: the agent's request + X-Mesh-Authentication: Biscuit
+ N->>S: mesh stream with the caller's Biscuit
+ Note over S: verifies the Biscuit and the policy again
+```
+
+- The workload holds no credential. The enforcement point of the runtime gets
+ the Biscuit and attaches it to each request. The agent sees one
+ unauthenticated MCP endpoint.
+- The `--workload-issuer` and `--allowed-audiences` flags on the control plane
+ make the trust decision. The control plane authenticates every identity
+ that the issuer signs for that audience. Only the `bindings` define what
+ the identity can do.
+- A binding member is `user:<sub>`. The member can contain one `*`, at the
+ start or at the end. For example,
+ `user:spiffe://openshell.local/openshell/sandbox/*` binds every sandbox of
+ one OpenShell gateway.
+- A node publishes a service only when its role lists the service in
+ `allowed_publish`.
+- The serving node verifies the Biscuit again. A Biscuit is bound to the node
+ that obtained it. A copy of the Biscuit does not work from another node.
+- The Biscuit carries the roles that the control plane gave it at mint time.
+ The serving node allows a call when the roles of the Biscuit or the current
+ policy grant the call. A new binding takes effect at the next policy sync
+ of the nodes, within seconds. A removed binding takes effect when the
+ runtime replaces the cached Biscuit of the caller. For a Substrate actor,
+ this takes up to 60 s (the cache of the provider). For an OpenShell sandbox
+ at `0.1.4`, the measured time is about 3 minutes, and
+ `cache_ttl_seconds: 60` in the profile did not shorten it.
+
+| | OpenShell sandbox | Substrate actor |
+|---|---|---|
+| Identity | SPIRE JWT-SVID, `spiffe://openshell.local/openshell/sandbox/<id>` | actor X.509 certificate, `spiffe://substrate-actor.local/actor/<atespace>/<name>` |
+| Who exchanges it | the supervisor, with a provider profile (`token_grant`, `jwt-spiffe` assertion, RFC 7523) | the egress gateway, through a `CredentialProvider` that answers `ate-secret://agentmesh/mesh/biscuit` |
+| Where the mesh endpoint is allowed | `NetworkEndpoint` in the sandbox policy and `allowed_ips` in the provider profile | `EgressPolicy.rules[].http.effects.replaceHeaders[].credentialUri` |
+| What the agent sees | `http://host.openshell.internal:8090/mcp` (Docker) or the edge Service (Kubernetes) | `http://edge-node.agentmesh.svc.cluster.local.:8080/mcp` |
+
+### 1.3 Versions
+
+| Component | Version |
+|---|---|
+| Agent Mesh (`agentmesh-one`, `agentmesh-node`) | `v0.1.1` |
+| OpenShell | `0.1.3` (Docker driver), `0.1.4` (Kubernetes driver, Helm chart) |
+| Agent Sandbox (`agents.x-k8s.io`) | `v1.0.7` |
+| SPIRE (SPIFFE hardened charts) | `0.29.0` (CRDs `0.5.0`) |
+| Agent Substrate | `v0.5.0` |
+| GKE | `1.36.4`, Dataplane V2 |
+
+### 1.4 The examples
+
+| Directory | Runtime | Driver | Cluster |
+|---|---|---|---|
+| [`development/examples/openshell-sandbox`](https://github.com/google/agentmesh/tree/main/development/examples/openshell-sandbox) | OpenShell | Docker | none (Docker Compose) |
+| [`development/examples/openshell-kubernetes`](https://github.com/google/agentmesh/tree/main/development/examples/openshell-kubernetes) | OpenShell | Kubernetes | kind or GKE |
+| [`development/examples/substrate-actor`](https://github.com/google/agentmesh/tree/main/development/examples/substrate-actor) | Substrate | - | kind or GKE |
+
+Each example has an edge node, an office node and a policy. The office node
+publishes a calculator as `mcp://calc` and as `mcp://payroll`. The policy
+gives the sandbox role access to `mcp://calc` and not to `mcp://payroll`.
+`make call` shows the tools that the workload can see. Then it calls `calc`
+and calls `payroll`. The expected result is the same in each example:
+
+```text
+mcp://calc/add(2, 3) = 5.0
+mcp://payroll/add(2, 3) -> refused: ... biscuit: denied by no matching policies
+```
+
+## 2. The hub
+
+The hub is one `agentmesh-one`. It is the control plane and the router for
+every cluster in this tutorial.
+
+### 2.1 In the cluster
+
+For one cluster, run the hub in the cluster. The examples do this with
+`HUB=local`, which is the default. No other step is necessary.
+
+### 2.2 On Cloud Run
+
+For more than one cluster, run the hub at a location that every cluster can
+reach. This tutorial uses Cloud Run. Follow the
+[Cloud Run guide](/docs/guides/cloud-run/) for the service. Then do these
+steps:
+
+1. Create the signing keys and the policy as secrets. You own the keys. The
+ control plane installs them from the file and does not rotate them. To
+ rotate the keys, edit the secret. The Cloud Run guide has the procedure.
+
+ ```bash
+ openssl genpkey -algorithm ed25519 -out ed25519-v1.pem
+ openssl genpkey -algorithm ec -pkeyopt ec_paramgen_curve:P-256 -out p256-v1.pem
+ cat ed25519-v1.pem p256-v1.pem | gcloud secrets create mesh-signing-keys --data-file=-
+ gcloud secrets create mesh-policy --data-file=policy.json
+ SA=$(gcloud projects describe <project> --format='value(projectNumber)')[email protected]
+ for s in mesh-signing-keys mesh-policy; do
+ gcloud secrets add-iam-policy-binding "$s" --member="serviceAccount:$SA" --role=roles/secretmanager.secretAccessor
+ done
+ ```
+
+2. Mount both secrets, state the public URL, and pass the issuers of both
+ runtimes. The public URL is the router address that members dial and the
+ OIDC issuer that the control plane signs with. On Cloud Run, a list in
+ `--args` needs the `^;^` delimiter:
+
+ ```bash
+ gcloud run services update <hub> --region <region> \
+ --set-secrets=/etc/agentmesh/keys/signing-keys.pem=mesh-signing-keys:latest,/etc/agentmesh/policy.json=mesh-policy:latest \
+ --args="^;^--data-dir=/data;--port=8080;--external-url=https://<hub>-<project number>.<region>.run.app;--signing-key-path=/etc/agentmesh/keys/signing-keys.pem;--policy-file=/etc/agentmesh/policy.json;--allowed-audiences=agentmesh;--workload-issuer=<openshell issuer>,<substrate issuer>"
+ ```
+
+ Sections 4 and 6 define each issuer. The control plane starts before an
+ issuer answers. It accepts the tokens of that issuer after the issuer
+ starts to answer.
+
+ To change the policy, run `gcloud secrets versions add mesh-policy
+ --data-file=policy.json` and deploy a new revision. For an immediate
+ change, `POST /policies` with the admin token. That change does not
+ survive a revision.
+
+3. Write the `AGENTMESH_TOKEN` of the hub to `run/join-token` in each example
+ directory that you use. Write the `AGENTMESH_ADMIN_TOKEN` to
+ `run/admin-token` in the same directories.
+
+A new revision keeps the keys and the policy. The nodes start with
+`--bootstrap-token-path`. A node that the new revision does not know enrolls
+again automatically.
+
+Give the service a `--timeout` of 600 seconds or less. The WebSocket from a
+member to the router lives until the timeout. After a new revision, a member
+stays on the old instance until its connection ends. Until then, the two
+instances have separate directories.
+
+## 3. OpenShell on Kubernetes you manage
+
+OpenShell runs from its Helm chart. Each sandbox is an `agents.x-k8s.io`
+`Sandbox` with a supervisor pod next to it. SPIRE gives the supervisor pod a
+SPIFFE identity through one `ClusterSPIFFEID`. The supervisor exchanges that
+identity for a Biscuit at the edge node. One chart value
+(`provider_spiffe_workload_api_socket_path`) and one provider profile connect
+OpenShell to the mesh. The example does not modify OpenShell.
+
+You need `kubectl`, `helm`, `docker`, `kind`, the `openshell` CLI (`0.1.3`
+or later), `python3` and `curl`.
+
+```bash
+cd development/examples/openshell-kubernetes
+make cluster # a kind cluster
+make up # Agent Sandbox, SPIRE, the mesh with the hub in the cluster, OpenShell
+make sandbox NAME=demo # prints spiffe://openshell.local/openshell/sandbox/<id>
+make call NAME=demo # refused: no role binds the identity yet
+make bind NAME=demo # adds the member to the sandbox role and pushes the policy
+make call NAME=demo # calc allowed, payroll refused
+make netpol-check NAME=demo # the workload pod cannot reach the edge node; an unconfined pod can
+make rogue # a second sandbox, not bound: refused
+make unbind NAME=demo # removes the member; refused within about 3 minutes
+make down # OpenShell, the mesh, SPIRE
+make clean # the kind cluster
+```
+
+`make bind` edits the `sandbox` binding in `policy.json` and pushes the
+policy. To bind every sandbox of this gateway, set the member to
+`user:spiffe://openshell.local/openshell/sandbox/*` instead.
+
+Expected output of `make call` after `make bind`:
+
+```text
+== tools the mesh offers sandbox demo
+ 12D3KooWM5k2n5zm… mcp://calc/add Return a + b.
+ 12D3KooWM5k2n5zm… mcp://calc/multiply Return a * b.
+== mcp://calc, allowed for the sandbox role
+mcp://calc/add(2, 3) = 5.0
+== mcp://payroll, not in the sandbox role
+ (no tool of mcp://payroll is visible to this sandbox; calling the office node directly)
+mcp://payroll/add(2, 3) -> refused: auth rejected by 12D3KooWM5k2…: biscuit: denied by no matching policies
+```
+
+The hub logs one line for each credential that it mints:
+
+```text
+Border Crossing {"endpoint": "/token/exchange", "decision": "allow",
+ "subject": "spiffe://openshell.local/openshell/sandbox/2c399868-…", "roles": ["sandbox"]}
+```
+
+The CNI must enforce `NetworkPolicy`. `make netpol-check` creates a pod
+without a sandbox. It shows that this pod reaches the edge node and that the
+workload pod does not.
+
+To run the same example with the Docker driver and no cluster, use
+`development/examples/openshell-sandbox` and the
+[OpenShell guide](/docs/guides/openshell/).
+
+## 4. OpenShell on GKE
+
+The same example runs on GKE with the hub on Cloud Run. Three items differ
+from kind.
+
+**Node image.** The sandbox runtime of OpenShell needs Landlock ABI v3. The
+Container-Optimized OS kernel has no `CONFIG_SECURITY_LANDLOCK`. On that
+kernel, the workload pod exits with `Landlock ABI probe … kernel lacks
+CONFIG_SECURITY_LANDLOCK`. Create the node pool for the sandboxes with
+`--image-type UBUNTU_CONTAINERD`.
+
+**Service CIDR.** GKE gives ClusterIPs from `34.118.224.0/20`, which is
+outside RFC 1918. The example renders the `allowed_ips` of the provider
+profile from the ClusterIP of the edge Service. A wrong list shows in the
+supervisor log as `NET:REFUSE … policy_dns_no_valid_address`.
+
+**Issuer.** The hub must fetch the OIDC discovery document and the keys of
+SPIRE. Expose the SPIRE OIDC discovery provider on a reserved address. Name
+that address as the issuer.
+
+```bash
+gcloud container clusters create <cluster> --zone <zone> --enable-dataplane-v2 --num-nodes 2
+gcloud container node-pools create ubuntu-pool --cluster <cluster> --zone <zone> \
+ --image-type UBUNTU_CONTAINERD --num-nodes 2
+gcloud compute addresses create spire-oidc --region <region>
+IP=$(gcloud compute addresses describe spire-oidc --region <region> --format='value(address)')
+
+cd development/examples/openshell-kubernetes
+export KUBE_CONTEXT=gke_<project>_<zone>_<cluster> OPENSHELL_GATEWAY=openshell-gke
+make prereqs OIDC_SERVICE_TYPE=LoadBalancer OIDC_LOAD_BALANCER_IP=$IP JWT_ISSUER=http://$IP
+# hub: add http://$IP to --workload-issuer (Section 2.2), then
+cp <join token> run/join-token && cp <admin token> run/admin-token
+docker build -t <registry>/calc-mcp:v1 ../calc-mcp && docker push <registry>/calc-mcp:v1
+docker build -t <registry>/sandbox:v1 -f Dockerfile.sandbox . && docker push <registry>/sandbox:v1
+make mesh HUB=https://<hub>.run.app CALC_MCP_IMAGE=<registry>/calc-mcp:v1
+make openshell HUB=https://<hub>.run.app SANDBOX_IMAGE=<registry>/sandbox:v1
+make sandbox NAME=demo && make bind NAME=demo HUB=https://<hub>.run.app && make call NAME=demo
+```
+
+`OPENSHELL_GATEWAY` is the name under which the `openshell` CLI stores the
+gateway of this cluster. Use one name per cluster. Each target selects its
+own name.
+
+The issuer is plain HTTP on a public address. Any server that answers at
+that address can mint identities. Put TLS in front of the issuer before you
+use it for more than a test.
+
+`make bind` with `HUB=<url>` pushes the policy to the hub with the admin
+token. To keep the change across a revision, also add the policy as a new
+version of the `mesh-policy` secret.
+
+The result is the same as in Section 3. The `Border Crossing` line of the
+hub is in Cloud Logging.
+
+## 5. Substrate on Kubernetes you manage
+
+The egress gateway of Substrate injects credentials that a
+`CredentialProvider` returns. The example runs a provider that answers
+`ate-secret://agentmesh/mesh/biscuit` with a Biscuit for the SPIFFE ID of the
+calling actor. To get that Biscuit, the provider acts as a workload identity
+issuer for the mesh. It signs a JWT for the actor and exchanges the JWT at
+the edge node, as the OpenShell supervisor does. The `EgressPolicy` of the
+actor lets the actor reach the edge node. The `EgressPolicy` also names the
+credential for the `X-Mesh-Authentication` header.
+
+You need `docker`, `kind`, `ko`, `kubectl`, Go, `python3`, and a checkout of
+Substrate at `v0.5.0`.
+
+```bash
+cd development/examples/substrate-actor
+make cluster SUBSTRATE_DIR=~/src/substrate # kind, Substrate from source, egress gateway pointed at the provider
+make up # the mesh with the hub in the cluster, and the provider
+make actor NAME=demo # an actor with its EgressPolicy
+make call NAME=demo # tools, calc allowed, payroll refused
+make rogue # an actor the policy does not bind: refused
+make down # the actors and the agentmesh namespace
+make clean # the kind cluster
+```
+
+The policy is in `manifests/mesh.yaml.tmpl`. A binding gives the `actor`
+role to `user:spiffe://substrate-actor.local/actor/ate-demo-sandbox/demo`. To
+admit the rogue actor, add
+`user:spiffe://substrate-actor.local/actor/ate-demo-sandbox/rogue` to the
+members and run `make policy`.
+
+Expected output of `make call`:
+
+```text
+== tools the mesh offers actor demo
+ 12D3KooWEKsFc3PM… mcp://calc/add Return a + b.
+ 12D3KooWEKsFc3PM… mcp://calc/multiply Return a * b.
+== mcp://calc, allowed for the actor role
+mcp://calc/add(2, 3) = 5.0
+== mcp://payroll, not in the actor role
+ (no tool of mcp://payroll is visible to this actor; calling the office node directly)
+mcp://payroll/add(2, 3) -> refused: auth rejected by 12D3KooWEKsFc3PM…: biscuit: denied by no matching policies
+```
+
+Two properties of Substrate `v0.5.0` matter here:
+
+- `--credential-provider` needs the envoy dataplane, which is the default.
+- The DNS relay of the actor answers `REFUSED` for a name outside the
+ `EgressPolicy`. With `ndots:5` and the musl resolver (Alpine images), the
+ resolver expands a relative name with the search list first. Those
+ expansions get `REFUSED` and the lookup fails. Use absolute names with a
+ trailing dot, for example `edge-node.agentmesh.svc.cluster.local.`. The
+ example does this.
+
+### 5.1 With the hub on Cloud Run
+
+With `HUB=<url>`, the hub must fetch the discovery document and the JWKS of
+the provider. A kind cluster cannot serve them. The provider signs with a key
+that you keep. The two documents are static files at a public URL:
+
+```bash
+openssl genpkey -algorithm ec -pkeyopt ec_paramgen_curve:P-256 -out provider-p256.pem
+export HUB=https://<hub>.run.app ISSUER=https://storage.googleapis.com/<bucket>/substrate PROVIDER_KEY_PATH=$PWD/provider-p256.pem
+make issuer-docs # writes run/issuer/.well-known/openid-configuration and run/issuer/keys
+gcloud storage cp -r run/issuer/.well-known run/issuer/keys gs://<bucket>/substrate/
+# hub: add $ISSUER to --workload-issuer (Section 2.2), then
+cp <join token> run/join-token && cp <admin token> run/admin-token
+make up && make actor NAME=demo && make call NAME=demo
+```
+
+To rotate the key, do these steps. Create a new key. Publish a JWKS with
+both keys. Replace the Secret `credential-provider-signing-key`. Then remove
+the old key from the JWKS.
+
+## 6. Substrate on GKE
+
+The tooling of Substrate creates the cluster and installs Substrate. The
+example adds the mesh. Run these commands in the Substrate checkout:
+
+```bash
+cp hack/ate-dev-env.sh.example .ate-dev-env.sh # set PROJECT_ID, GCE_REGION, CLUSTER_LOCATION, CLUSTER_NAME, KO_DOCKER_REPO
+source .ate-dev-env.sh
+gcloud auth application-default set-quota-project $PROJECT_ID
+go run ./tools/setup-gcp bootstrap # APIs, GKE cluster, GCS bucket, IAM (about 8 minutes)
+kubectl apply -f <agentmesh>/development/examples/substrate-actor/manifests/namespace.yaml
+./hack/install-ate.sh --deploy-ate-system --deploy-demo-sandbox \
+ --credential-provider='{"name":"agentmesh","address":"credential-provider.agentmesh.svc:50051"}'
+```
+
+The `namespace.yaml` step comes before the install because the egress
+gateway resolves the Service of the provider at start. `atenet-egress` stays
+`1/3` until the provider runs.
+
+Then run these commands in the example directory, with the hub, the issuer
+and the key of Section 5.1:
+
+```bash
+cd development/examples/substrate-actor
+export KUBE_CONTEXT=gke_<project>_<zone>_<cluster> KO_DOCKER_REPO=<registry>
+docker build -t <registry>/calc-mcp:v1 ../calc-mcp && docker push <registry>/calc-mcp:v1
+export CALC_MCP_IMAGE=<registry>/calc-mcp:v1
+export HUB=https://<hub>.run.app ISSUER=https://storage.googleapis.com/<bucket>/substrate PROVIDER_KEY_PATH=$PWD/provider-p256.pem
+cp <join token> run/join-token && cp <admin token> run/admin-token
+make up # about 30 s; atenet-egress goes to 3/3
+make actor NAME=demo && make call NAME=demo
+```
+
+The result is the same as in Section 5. The actor runs in gVisor. On its
+first call, it resumes from the golden snapshot of the template.
+
+Two Substrate clusters on one hub present the same actor identities.
+`spiffe://substrate-actor.local/actor/<atespace>/<name>` is the trust domain
+of Substrate, not of the cluster. Run one Substrate cluster per hub, or use a
+different atespace name in each cluster.
+
+## 7. Both runtimes on one mesh
+
+With OpenShell (Section 3 or 4) and Substrate (Section 5 or 6) on the Cloud
+Run hub, one policy binds both identities to one role:
+
+```json
+{
+ "roles": [
+ {"name": "mesh:role:node", "allowed_services": ["*"], "allowed_publish": ["*"], "allowed_targets": ["*"], "allowed_labels": ["*"]},
+ {"name": "mesh:role:router", "allowed_services": ["*"], "allowed_targets": ["*"]},
+ {"name": "sandbox", "allowed_services": ["system://mesh.catalog", "mcp://calc"], "allowed_targets": ["*"]}
+ ],
+ "bindings": [
+ {"role": "sandbox", "members": [
+ "user:spiffe://openshell.local/openshell/sandbox/*",
+ "user:spiffe://substrate-actor.local/actor/ate-demo-sandbox/demo"
+ ]}
+ ]
+}
+```
+
+Push the policy as in Section 2.2. Then, from either side, `make call` lists
+two providers of `mcp://calc`, the office node in each cluster. The call
+goes to the first provider by peer ID. `PEER=<prefix>` selects one provider:
+
+```bash
+cd development/examples/openshell-kubernetes
+make call NAME=demo PEER=<peer id of the office node in the Substrate cluster>
+```
+
+```text
+== tools the mesh offers sandbox demo
+ 12D3KooWDV7WA57x… mcp://calc/add Return a + b.
+ 12D3KooWJozR2PAJ… mcp://calc/add Return a + b.
+== mcp://calc, allowed for the sandbox role
+ calling provider 12D3KooWJozR2PAJ…
+mcp://calc/add(2, 3) = 5.0
+== mcp://payroll, not in the sandbox role
+ calling provider 12D3KooWJozR2PAJ…
+mcp://payroll/add(2, 3) -> refused: auth rejected by 12D3KooWJozR2PAJ…: biscuit: denied by no matching policies
+```
+
+The office node that served the call logs the identity of the caller:
+
+```text
+Audit Traceability {"target": "mcp://calc", "decision": "allow",
+ "user": "spiffe://openshell.local/openshell/sandbox/69d290b9-…", "role": "sandbox", "actor_node": "12D3KooWD1Lm…"}
+```
+
+The other direction works in the same way. An actor in the Substrate cluster
+calls `mcp://calc` that the office node in the OpenShell cluster serves. That
+node logs `"user": "spiffe://substrate-actor.local/actor/ate-demo-sandbox/demo"`.
+
+## 8. Move a workload
+
+A workload moves from OpenShell to Substrate, or back, without a change to
+the agent image. The image speaks HTTP to one MCP endpoint and holds no
+credential. The runtime supplies the identity. The mesh supplies the
+permissions.
+
+| OpenShell | Substrate | Mesh |
+|---|---|---|
+| sandbox `image` | `ActorTemplate.spec.image` | no change |
+| `NetworkEndpoint` for the mesh endpoint | `EgressPolicy` rule for `edge-node.agentmesh.svc.cluster.local.` | no change |
+| provider profile `token_grant` for the mesh endpoint | `EgressPolicy` `replaceHeaders` with `ate-secret://agentmesh/mesh/biscuit` | no change |
+| `spiffe://openshell.local/openshell/sandbox/<id>` | `spiffe://substrate-actor.local/actor/<atespace>/<name>` | the member in the binding |
+| MCP endpoint `http://host.openshell.internal:8090/mcp` or the edge Service | `http://edge-node.agentmesh.svc.cluster.local.:8080/mcp` | the same tools, the same refusals |
+
+To move one workload:
+
+1. Run the OpenShell sandbox on the mesh (Section 3 or 4). Its role has the
+ tools that it needs.
+2. Create an `ActorTemplate` from the image of the sandbox. Create an
+ `EgressPolicy` like the one in `substrate-actor/manifests`. `make actor`
+ applies both.
+3. Add the identity of the actor to the same role. Both identities hold the
+ role now. You can start and test the actor while the sandbox continues to
+ run.
+4. Remove the identity of the sandbox from the role. The mesh refuses the
+ sandbox when the runtime replaces its cached Biscuit. With OpenShell
+ `0.1.4`, this happens about 3 minutes later. This step is the cutover.
+5. To roll back, add the identity of the sandbox again. The mesh admits the
+ sandbox at the next policy sync of the nodes, within seconds.
+
+The same steps move a workload from Substrate to OpenShell. A removed actor
+binding takes effect within 60 s.
+
+Each of steps 3 to 5 is one policy push. No component restarts.
+
+## 9. A Google API from either runtime
+
+The same policy gives both runtimes a Google API. The egress node that
+serves the API exchanges the token of the control plane at Google STS. It
+then injects the Google access token. The workload holds no Google
+credential. The setup on the Google side (Workload Identity Pool, provider,
+service account) and the `egress` entry are in [Google APIs with a federated
+credential](/docs/guides/google-cloud-credentials/). With the pool and the
+service account from that guide, add this to the policy of Section 7:
+
+```json
+"roles": [
+ {"name": "sandbox",
+ "allowed_services": ["system://mesh.catalog", "mcp://calc", "egress://aiplatform.googleapis.com"],
+ "allowed_targets": ["*"],
+ "http": [{"service": "egress://aiplatform.googleapis.com", "methods": ["POST"],
+ "paths": ["/v1/projects/<project>/locations/us-central1/publishers/google/models/*"]}]}
+],
+"egress": [
+ {"name": "aiplatform.googleapis.com", "served_by": ["site=office"],
+ "broker": {"oidc_federation": {
+ "token_endpoint": "https://sts.googleapis.com/v1/token",
+ "audience": "//iam.googleapis.com/projects/<project number>/locations/global/workloadIdentityPools/agentmesh/providers/hub-cp",
+ "impersonate": "mesh-gemini@<project>.iam.gserviceaccount.com",
+ "scopes": ["https://www.googleapis.com/auth/cloud-platform"]}}}
+]
+```
+
+Push the policy. Each office node logs `[Egress] Serving egress://aiplatform.googleapis.com`.
+The issuer URL of the hub must be `https`; `--external-url` in Section 2.2
+sets it.
+
+The workload reaches the destination through its edge node at
+`/mesh/<office peer>/egress/aiplatform.googleapis.com/...`. The OpenShell
+provider profile in the example allows that path (`/mesh/**`). The Substrate
+`EgressPolicy` has no path restriction. Then run this command from each
+example directory:
+
+```bash
+make gemini NAME=demo GCP_PROJECT=<project>
+```
+
+```text
+== POST gemini-2.5-flash:generateContent, granted to the sandbox role
+ answer: Five
+== GET on the model, a method the grant does not list
+ HTTP 403 agentmesh-node; error=http_request_denied
+== POST to another Vertex AI path, outside the granted paths
+ HTTP 403 agentmesh-node; error=http_request_denied
+```
+
+The Substrate example prints the same results. Its status lines come from
+`wget` inside the actor (`HTTP/1.1 403 Forbidden`).
+
+The office node that served the call logs each decision with the identity of
+the caller, as in Section 7. The log line has
+`"target": "egress://aiplatform.googleapis.com"`, the method and the path.
+The audit log of Google shows the impersonation of the service account, not
+a key.
+
+### 9.1 An agent on Agent Engine
+
+`aiplatform.googleapis.com` is also the host of Agent Engine. A workload
+calls an agent that runs there in the same way. Deploy a minimal agent with
+the Vertex AI SDK:
+
+```python
+# deploy.py: a class with a query method is an agent.
+import vertexai
+from vertexai import agent_engines
+
+class Calculator:
+ def set_up(self):
+ pass
+
+ def query(self, a: float, b: float) -> dict:
+ return {"sum": a + b, "served_by": "agent-engine"}
+
+vertexai.init(project="<project>", location="us-central1", staging_bucket="gs://<staging bucket>")
+engine = agent_engines.create(Calculator(), display_name="mesh-calculator")
+print(engine.resource_name) # projects/<project number>/locations/us-central1/reasoningEngines/<id>
+```
+
+```bash
+pip install "google-cloud-aiplatform[agent_engines]"
+python3 deploy.py # a few minutes
+```
+
+Add the engine path to the `paths` of the `http` entry. A role has one
+`http` entry per service, so both paths go in the same list:
+
+```json
+"paths": ["/v1/projects/<project>/locations/us-central1/publishers/google/models/*",
+ "/v1/projects/<project>/locations/us-central1/reasoningEngines/*"]
+```
+
+Push the policy, then run `make gemini` with the engine id:
+
+```bash
+make gemini NAME=demo GCP_PROJECT=<project> AGENT_ENGINE=<id>
+```
+
+```text
+== POST reasoningEngines/<id>:query, an agent on Agent Engine
+ output: {"sum": 42, "served_by": "agent-engine"}
+```
+
+The service account `mesh-gemini` needs `roles/aiplatform.user` on the
+project, which the guide grants. A `GET` on the engine resource is refused
+like the other requests outside the grant.
+
+## 10. Clean up
+
+```bash
+# kind
+cd development/examples/openshell-kubernetes && make down && make clean
+cd development/examples/substrate-actor && make down && make clean
+# GKE and Cloud Run
+gcloud container clusters delete <openshell cluster> --zone <zone>
+gcloud container clusters delete <substrate cluster> --zone <zone>
+gcloud run services delete <hub> --region <region>
+gcloud compute addresses delete spire-oidc --region <region>
+gcloud secrets delete mesh-policy
+gcloud storage rm -r gs://<bucket>/substrate
+gcloud iam service-accounts delete mesh-gemini@<project>.iam.gserviceaccount.com
+gcloud iam workload-identity-pools delete agentmesh --location global
+python3 -c 'import vertexai; from vertexai import agent_engines; vertexai.init(project="<project>", location="us-central1"); agent_engines.delete("<engine resource name>", force=True)'
+```
+
+The Substrate bootstrap also created a snapshot bucket and three Cloud
+Monitoring dashboards. `go run ./tools/setup-gcp --help` lists the resources.
diff --git a/site/content/docs/guides/openshell.md b/site/content/docs/guides/openshell.md
index 9aec11d0..68465ed2 100644
--- a/site/content/docs/guides/openshell.md
+++ b/site/content/docs/guides/openshell.md
@@ -67,9 +67,9 @@ flowchart LR
gateway keeps its state in `/var/lib/openshell`).
- The `openshell` CLI, 0.1.3 or later, from the
[OpenShell installation page](https://docs.openshell.ai/about/installation).
-- `agentmesh-one` and `agentmesh-node` images newer than v0.1.0-rc.9, which
- is when the node's token endpoint learned the client assertion grant. The
- Compose project uses the `latest` images; set `AGENTMESH_VERSION` to pin a
+- `agentmesh-one` and `agentmesh-node` images at `v0.1.0` or newer, the first
+ release whose node token endpoint takes the client assertion grant. The
+ Compose project defaults to `v0.1.1`; set `AGENTMESH_VERSION` for another
release.
- `python3` and `curl`.
diff --git a/site/content/docs/guides/substrate.md b/site/content/docs/guides/substrate.md
index ea599e49..243e3b90 100644
--- a/site/content/docs/guides/substrate.md
+++ b/site/content/docs/guides/substrate.md
@@ -85,8 +85,8 @@ issuer directly. The last section shows that change.
- Docker, kind, ko, kubectl, Go and python3.
- A checkout of Agent Substrate; the example installs it from source with
Substrate's own scripts.
-- `agentmesh-one` and `agentmesh-node` images newer than v0.1.0-rc.9. The
- manifests use the `latest` images; set `AGENTMESH_VERSION` to pin a release.
+- `agentmesh-one` and `agentmesh-node` images at `v0.1.0` or newer. The
+ Makefile defaults to `v0.1.1`; set `AGENTMESH_VERSION` for another release.
```bash
cd development/examples/substrate-actor