Skip to content

Agent Mesh

docs,examples: a Substrate actor on the mesh through a credential provider

Antonio Ojeae83c34456abb
diff --git a/development/examples/substrate-actor/Makefile b/development/examples/substrate-actor/Makefile
new file mode 100644
index 00000000..87ca6f4c
--- /dev/null
+++ b/development/examples/substrate-actor/Makefile
@@ -0,0 +1,126 @@
+# An Agent Substrate actor on the mesh under its own identity.
+#
+# make cluster kind cluster with Substrate; the egress gateway is
+# pointed at the Agent Mesh credential provider
+# make up the mesh and the provider, in namespace agentmesh
+# make actor NAME=demo one sandbox actor with an egress policy
+# make call NAME=demo from inside the actor: tools, calc allowed, payroll refused
+# make rogue an actor the mesh policy does not bind: refused
+# make policy push the mesh policy after editing manifests/mesh.yaml.tmpl
+# make down remove the actors, the mesh and the provider
+# make clean delete the kind cluster
+#
+# Needs: docker, kind, ko, kubectl, go, python3, and a checkout of Agent
+# Substrate in SUBSTRATE_DIR (default ../../../../substrate). `make cluster`
+# builds Substrate from that checkout, which takes a while the first time.
+
+SHELL := /bin/bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+
+NAME ?= demo
+SUBSTRATE_DIR ?= $(abspath ../../../../substrate)
+KIND_CLUSTER_NAME ?= kind
+CTX := kind-$(KIND_CLUSTER_NAME)
+KUBECTL := kubectl --context $(CTX)
+KATE := kubectl-ate --context $(CTX)
+ATESPACE := ate-demo-sandbox
+TEMPLATE := sandbox-template
+TRUST_DOMAIN := substrate-actor.local
+ROUTER_PORT ?= 8000
+KO_DOCKER_REPO ?= localhost:5001
+
+AGENTMESH_VERSION ?= latest
+AGENTMESH_ONE_IMAGE ?= ghcr.io/google/agentmesh-one:$(AGENTMESH_VERSION)
+AGENTMESH_NODE_IMAGE ?= ghcr.io/google/agentmesh-node:$(AGENTMESH_VERSION)
+
+KUBECTL_CONTEXT := $(CTX)
+export KIND_CLUSTER_NAME KUBECTL_CONTEXT KO_DOCKER_REPO
+
+.PHONY: cluster up actor call rogue policy status down clean
+
+cluster:
+ @for t in docker kind ko kubectl go python3; do command -v $$t >/dev/null || { echo "$$t not found"; exit 1; }; done
+ @test -f "$(SUBSTRATE_DIR)/hack/install-ate-kind.sh" || { echo "SUBSTRATE_DIR=$(SUBSTRATE_DIR) is not an Agent Substrate checkout"; exit 1; }
+ kind get clusters 2>/dev/null | grep -qx "$(KIND_CLUSTER_NAME)" || (cd "$(SUBSTRATE_DIR)" && ./hack/create-kind-cluster.sh)
+ # The provider's Service must resolve when the gateway starts, so the
+ # namespace and Service exist before Substrate does; the pods come with `make up`.
+ $(KUBECTL) apply -f manifests/namespace.yaml
+ cd "$(SUBSTRATE_DIR)" && ./hack/install-ate-kind.sh --deploy-ate-system --deploy-demo-sandbox \
+ --credential-provider='{"name":"agentmesh","address":"credential-provider.agentmesh.svc:50051"}'
+ cd "$(SUBSTRATE_DIR)" && go install ./cmd/kubectl-ate
+ @echo "Substrate is up; next: make up"
+
+up:
+ $(KUBECTL) apply -f manifests/namespace.yaml
+ $(KUBECTL) -n agentmesh get secret agentmesh-tokens >/dev/null 2>&1 || \
+ $(KUBECTL) -n agentmesh create secret generic agentmesh-tokens \
+ --from-literal=join-token="mesh_tok_$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" \
+ --from-literal=admin-token="mesh_adm_$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" \
+ --from-literal=api-token="$$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')"
+ provider=$$(cd credential-provider && KO_DOCKER_REPO=$(KO_DOCKER_REPO)/agentmesh-credential-provider ko build --platform=linux/$$(go env GOARCH) --bare .)
+ calc=$$(docker build -q ../calc-mcp -t calc-mcp:local >/dev/null && kind load docker-image calc-mcp:local --name $(KIND_CLUSTER_NAME) >/dev/null && echo docker.io/library/calc-mcp:local)
+ for f in manifests/mesh.yaml.tmpl manifests/credential-provider.yaml.tmpl; do
+ AGENTMESH_ONE_IMAGE="$(AGENTMESH_ONE_IMAGE)" AGENTMESH_NODE_IMAGE="$(AGENTMESH_NODE_IMAGE)" \
+ CREDENTIAL_PROVIDER_IMAGE="$$provider" CALC_MCP_IMAGE="$$calc" \
+ envsubst '$$AGENTMESH_ONE_IMAGE $$AGENTMESH_NODE_IMAGE $$CREDENTIAL_PROVIDER_IMAGE $$CALC_MCP_IMAGE' < $$f | $(KUBECTL) apply -f -
+ done
+ $(KUBECTL) -n agentmesh rollout status deployment --timeout=180s
+ $(KUBECTL) -n ate-system rollout status deployment/atenet-egress --timeout=120s
+ @echo "mesh console: kubectl -n agentmesh port-forward svc/agentmesh-one 8070:8080 -> http://127.0.0.1:8070/console/"
+ @echo "next: make actor NAME=demo && make call NAME=demo"
+
+# The actor's identity is spiffe://$(TRUST_DOMAIN)/actor/$(ATESPACE)/<name>,
+# which Substrate derives from the certificate it gives the actor. The mesh
+# policy in manifests/mesh.yaml.tmpl says what that identity may call.
+actor:
+ $(KATE) create actor $(NAME) -a $(ATESPACE) --template $(TEMPLATE) 2>&1 | grep -v AlreadyExists || true
+ $(KATE) create egress-policy $(NAME) -a $(ATESPACE) -f egress-policy.yaml 2>&1 | grep -vE 'AlreadyExists|^ATESPACE|^$(ATESPACE) ' || true
+ # Resume is asynchronous and the first attempt can land before a worker
+ # is free; ask again until the actor reports RUNNING.
+ for _ in $$(seq 1 40); do
+ $(KATE) get actors -a $(ATESPACE) 2>/dev/null | grep "^$(ATESPACE) *$(NAME) " | grep -q ACTOR_STATE_RUNNING && break
+ $(KATE) resume actor $(NAME) -a $(ATESPACE) >/dev/null 2>&1 || true
+ sleep 3
+ done
+ $(KATE) get actors -a $(ATESPACE) | grep "^$(ATESPACE) *$(NAME) " | grep -q ACTOR_STATE_RUNNING || { echo "actor $(NAME) did not reach RUNNING"; exit 1; }
+ @echo "actor $(NAME) is spiffe://$(TRUST_DOMAIN)/actor/$(ATESPACE)/$(NAME); try: make call NAME=$(NAME)"
+
+call:
+ $(KUBECTL) -n ate-system port-forward service/atenet-router $(ROUTER_PORT):80 >/dev/null 2>&1 &
+ pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT; sleep 2
+ @echo "== tools the mesh offers actor $(NAME)"
+ ROUTER_PORT=$(ROUTER_PORT) ATESPACE=$(ATESPACE) python3 mcp.py tools $(NAME)
+ @echo "== mcp://calc, allowed for the actor role"
+ ROUTER_PORT=$(ROUTER_PORT) ATESPACE=$(ATESPACE) python3 mcp.py add $(NAME) calc 2 3
+ @echo "== mcp://payroll, not in the actor role"
+ ROUTER_PORT=$(ROUTER_PORT) ATESPACE=$(ATESPACE) python3 mcp.py add $(NAME) payroll 2 3 || true
+
+rogue:
+ $(MAKE) actor NAME=rogue
+ @echo "== rogue has an actor identity but the mesh policy binds it to no role"
+ $(MAKE) call NAME=rogue || true
+ @echo "== to let it in: add user:spiffe://$(TRUST_DOMAIN)/actor/$(ATESPACE)/rogue to the actor role in manifests/mesh.yaml.tmpl, then make policy"
+
+# Replaces the whole mesh policy. Nodes pick it up within 30s; an actor gets
+# its new roles when the provider's cached credential expires (--cache-ttl).
+policy:
+ $(KUBECTL) -n agentmesh port-forward svc/agentmesh-one 18070:8080 >/dev/null 2>&1 &
+ pf=$$!; trap 'kill $$pf 2>/dev/null' EXIT; sleep 2
+ curl -sS -X POST http://127.0.0.1:18070/policies \
+ -H "Authorization: Bearer $$($(KUBECTL) -n agentmesh get secret agentmesh-tokens -o jsonpath='{.data.admin-token}' | base64 -d)" \
+ -H 'Content-Type: application/json' \
+ --data-binary "$$($(KUBECTL) -n agentmesh get configmap mesh-config -o jsonpath='{.data.policy\.json}')"
+ @echo
+
+status:
+ $(KUBECTL) -n agentmesh get pods
+ $(KATE) get actors -a $(ATESPACE)
+ $(KUBECTL) -n ate-system logs deployment/atenet-egress -c ext-proc --tail=20 2>/dev/null | grep -iE 'agentmesh|credential|denied' || true
+
+down:
+ -for a in demo rogue; do $(KATE) suspend actor $$a -a $(ATESPACE) >/dev/null 2>&1; $(KATE) delete actor $$a -a $(ATESPACE) >/dev/null 2>&1; done
+ -$(KUBECTL) delete namespace agentmesh --wait=false
+
+clean:
+ kind delete cluster --name $(KIND_CLUSTER_NAME)
diff --git a/development/examples/substrate-actor/README.md b/development/examples/substrate-actor/README.md
new file mode 100644
index 00000000..93ddb0b9
--- /dev/null
+++ b/development/examples/substrate-actor/README.md
@@ -0,0 +1,31 @@
+# A Substrate actor on the mesh
+
+Everything the guide at `site/content/docs/guides/substrate.md` walks through,
+on a kind cluster:
+
+- Agent Substrate installed from a checkout, with its egress gateway pointed
+ at the Agent Mesh credential provider;
+- `credential-provider/`, a Substrate `CredentialProvider` that answers
+ `ate-secret://agentmesh/mesh/biscuit` with a mesh credential bound to the
+ calling actor's SPIFFE ID, by asserting that identity to the mesh as a
+ workload identity issuer;
+- `agentmesh-one`, an edge `agentmesh-node` the actors reach, and an office
+ node publishing a calculator as `mcp://calc` and `mcp://payroll`, all in
+ namespace `agentmesh`;
+- an `EgressPolicy` that lets the actor reach the edge node over plain HTTP and
+ has the gateway inject the credential as `X-Mesh-Authentication`.
+
+```bash
+make cluster SUBSTRATE_DIR=~/src/substrate # kind + Substrate (builds images; a few minutes)
+make up # the mesh and the provider
+make actor NAME=demo # a sandbox actor with the egress policy
+make call NAME=demo # tools, calc allowed, payroll refused
+make rogue # an actor the policy does not bind: refused
+make policy # push the policy after editing manifests/mesh.yaml.tmpl
+make down # the actors and the agentmesh namespace
+make clean # the kind cluster
+```
+
+The provider has its own Go module against the Substrate release it builds
+against (`credential-provider/go.mod`); `go test ./...` in that directory runs
+its unit tests without a cluster.
diff --git a/development/examples/substrate-actor/credential-provider/go.mod b/development/examples/substrate-actor/credential-provider/go.mod
new file mode 100644
index 00000000..a8d80d77
--- /dev/null
+++ b/development/examples/substrate-actor/credential-provider/go.mod
@@ -0,0 +1,17 @@
+module github.com/google/agentmesh/development/examples/substrate-actor/credential-provider
+
+go 1.27.0
+
+require (
+ github.com/agent-substrate/substrate v0.4.0
+ github.com/golang-jwt/jwt/v5 v5.3.1
+ google.golang.org/grpc v1.84.0
+)
+
+require (
+ golang.org/x/net v0.58.0 // indirect
+ golang.org/x/sys v0.47.0 // indirect
+ golang.org/x/text v0.41.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
+ google.golang.org/protobuf v1.36.12 // indirect
+)
diff --git a/development/examples/substrate-actor/credential-provider/go.sum b/development/examples/substrate-actor/credential-provider/go.sum
new file mode 100644
index 00000000..c81cd94e
--- /dev/null
+++ b/development/examples/substrate-actor/credential-provider/go.sum
@@ -0,0 +1,22 @@
+github.com/agent-substrate/substrate v0.4.0 h1:6TqMYb2ucahCcuPF5DfKaOmTRSoyEeHbfDkY+lLaLu4=
+github.com/agent-substrate/substrate v0.4.0/go.mod h1:ZYKQt3ZSTroeduZXmjEVsF2vjGyZryWDUQTH7hIvgIU=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
+golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0=
+google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0=
+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
diff --git a/development/examples/substrate-actor/credential-provider/main.go b/development/examples/substrate-actor/credential-provider/main.go
new file mode 100644
index 00000000..327b7d56
--- /dev/null
+++ b/development/examples/substrate-actor/credential-provider/main.go
@@ -0,0 +1,468 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Command credential-provider is an Agent Substrate credential provider that
+// gives every actor its own Agent Mesh credential.
+//
+// Substrate's egress gateway calls FetchSecret with the URI named in the
+// actor's EgressPolicy and the actor's SPIFFE ID, which the gateway verified
+// from the actor's client certificate. For ate-secret://agentmesh/mesh/biscuit
+// this provider signs a short-lived assertion that names that SPIFFE ID as the
+// subject, exchanges it at an agentmesh-node token endpoint (RFC 8693), and
+// returns the Biscuit the mesh minted. The control plane trusts this provider
+// as a workload identity issuer and binds roles to the actor's SPIFFE ID, so
+// the mesh policy, not the provider, decides what each actor may call.
+package main
+
+import (
+ "context"
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "crypto/sha256"
+ "crypto/tls"
+ "crypto/x509"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "encoding/pem"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "log/slog"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/signal"
+ "strings"
+ "sync"
+ "syscall"
+ "time"
+
+ "github.com/agent-substrate/substrate/pkg/proto/credproviderpb"
+ "github.com/golang-jwt/jwt/v5"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/status"
+)
+
+const (
+ grantTypeTokenExchange = "urn:ietf:params:oauth:grant-type:token-exchange"
+ tokenTypeJWT = "urn:ietf:params:oauth:token-type:jwt"
+ // Substrate credential URIs are ate-secret://<provider-class>/<provider-name>/<tail>.
+ credentialTail = "biscuit"
+ assertionLifetime = 10 * time.Minute
+ expirySkew = 30 * time.Second
+)
+
+var (
+ listenAddr = flag.String("listen-address", ":50051", "gRPC listen address for the Substrate CredentialProvider service")
+ httpAddr = flag.String("http-address", ":8080", "HTTP listen address for OIDC discovery, the JWKS and health")
+ serverBundle = flag.String("server-cred-bundle", "", "PEM private key and certificate chain this provider serves TLS with (required)")
+ clientCAFile = flag.String("client-ca-file", "", "CA bundle the egress gateway's client certificate must chain to (required)")
+ injectorIdentity = flag.String("injector-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-egress", "URI SAN the egress gateway's client certificate must carry")
+ providerName = flag.String("provider-name", "agentmesh", "provider class of the ate-secret:// URIs this provider serves")
+ instanceName = flag.String("instance-name", "mesh", "provider name segment of the ate-secret:// URIs this provider serves")
+ actorTrustDomain = flag.String("actor-trust-domain", "substrate-actor.local", "SPIFFE trust domain of actor identities")
+ nodeURL = flag.String("node-url", "", "agentmesh-node API whose /oauth/token mints the credential (required)")
+ issuer = flag.String("issuer", "", "iss of the assertions this provider signs; the control plane fetches <issuer>/.well-known/openid-configuration (required)")
+ audience = flag.String("audience", "agentmesh", "aud of the assertions; one of the control plane's --allowed-audiences")
+ cacheTTL = flag.Duration("cache-ttl", time.Minute, "longest a minted credential is reused; also how long a mesh policy change takes to reach an actor")
+)
+
+func main() {
+ flag.Parse()
+ logger := slog.New(slog.NewJSONHandler(os.Stderr, nil))
+ slog.SetDefault(logger)
+ if err := run(context.Background()); err != nil {
+ logger.Error("exiting", slog.Any("err", err))
+ os.Exit(1)
+ }
+}
+
+func run(ctx context.Context) error {
+ for name, value := range map[string]string{
+ "--server-cred-bundle": *serverBundle, "--client-ca-file": *clientCAFile,
+ "--node-url": *nodeURL, "--issuer": *issuer,
+ } {
+ if value == "" {
+ return fmt.Errorf("%s is required", name)
+ }
+ }
+ ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM)
+ defer stop()
+
+ signer, err := newSigner(*issuer, *audience)
+ if err != nil {
+ return err
+ }
+ provider := &provider{
+ name: *providerName,
+ instance: *instanceName,
+ trustDomain: *actorTrustDomain,
+ signer: signer,
+ node: strings.TrimRight(*nodeURL, "/"),
+ cacheTTL: *cacheTTL,
+ http: &http.Client{Timeout: 15 * time.Second},
+ cache: map[string]*cached{},
+ }
+
+ creds, err := serverCredentials(*serverBundle, *clientCAFile, *injectorIdentity)
+ if err != nil {
+ return err
+ }
+ grpcSrv := grpc.NewServer(grpc.Creds(creds))
+ credproviderpb.RegisterCredentialProviderServer(grpcSrv, provider)
+ lis, err := (&net.ListenConfig{}).Listen(ctx, "tcp", *listenAddr)
+ if err != nil {
+ return fmt.Errorf("listen %s: %w", *listenAddr, err)
+ }
+
+ httpSrv := &http.Server{Addr: *httpAddr, Handler: signer.handler(), ReadHeaderTimeout: 5 * time.Second}
+ errs := make(chan error, 2)
+ go func() { errs <- grpcSrv.Serve(lis) }()
+ go func() { errs <- httpSrv.ListenAndServe() }()
+ slog.Info("serving", slog.String("grpc", *listenAddr), slog.String("http", *httpAddr),
+ slog.String("issuer", *issuer), slog.String("node", provider.node), slog.String("uri", provider.uri()))
+
+ select {
+ case <-ctx.Done():
+ case err := <-errs:
+ if err != nil && !errors.Is(err, http.ErrServerClosed) {
+ return err
+ }
+ }
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ grpcSrv.GracefulStop()
+ return httpSrv.Shutdown(shutdownCtx)
+}
+
+// provider answers FetchSecret for one URI with a per-actor mesh credential.
+type provider struct {
+ credproviderpb.UnimplementedCredentialProviderServer
+ name string
+ instance string
+ trustDomain string
+ signer *signer
+ node string
+ cacheTTL time.Duration
+ http *http.Client
+
+ mu sync.Mutex
+ cache map[string]*cached
+}
+
+type cached struct {
+ mu sync.Mutex
+ token []byte
+ expires time.Time
+}
+
+type actorRef struct{ atespace, name string }
+
+// FetchSecret returns the Biscuit for the actor the gateway attested. The URI
+// must be ate-secret://<provider-name>/<instance-name>/biscuit; anything else
+// is not served.
+func (p *provider) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecretRequest) (*credproviderpb.FetchSecretResponse, error) {
+ if err := p.checkURI(req.GetUri()); err != nil {
+ return nil, status.Error(codes.NotFound, err.Error())
+ }
+ actor, err := p.parseActor(req.GetActorSpiffeId())
+ if err != nil {
+ return nil, status.Error(codes.PermissionDenied, err.Error())
+ }
+ token, err := p.credential(ctx, req.GetActorSpiffeId(), actor)
+ if err != nil {
+ return nil, err
+ }
+ return &credproviderpb.FetchSecretResponse{OpaqueBytes: token}, nil
+}
+
+func (p *provider) uri() string {
+ return "ate-secret://" + p.name + "/" + p.instance + "/" + credentialTail
+}
+
+func (p *provider) checkURI(raw string) error {
+ u, err := url.Parse(raw)
+ if err != nil {
+ return fmt.Errorf("credential URI: %w", err)
+ }
+ if u.Scheme != "ate-secret" || u.Host != p.name || u.Path != "/"+p.instance+"/"+credentialTail || u.RawQuery != "" {
+ return fmt.Errorf("credential URI %q is not %s", raw, p.uri())
+ }
+ return nil
+}
+
+// parseActor accepts spiffe://<trust-domain>/actor/<atespace>/<name>, the ID
+// Substrate's gateway derives from the actor's certificate.
+func (p *provider) parseActor(id string) (actorRef, error) {
+ u, err := url.Parse(id)
+ if err != nil {
+ return actorRef{}, fmt.Errorf("actor SPIFFE ID: %w", err)
+ }
+ parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/")
+ if u.Scheme != "spiffe" || u.Host != p.trustDomain || len(parts) != 3 || parts[0] != "actor" || parts[1] == "" || parts[2] == "" {
+ return actorRef{}, fmt.Errorf("actor SPIFFE ID %q is not spiffe://%s/actor/<atespace>/<name>", id, p.trustDomain)
+ }
+ return actorRef{atespace: parts[1], name: parts[2]}, nil
+}
+
+// credential returns the cached Biscuit for subject, minting one when the
+// cache has none or the entry has aged out. Concurrent requests for one actor
+// share a single mint.
+func (p *provider) credential(ctx context.Context, subject string, actor actorRef) ([]byte, error) {
+ p.mu.Lock()
+ entry, ok := p.cache[subject]
+ if !ok {
+ entry = &cached{}
+ p.cache[subject] = entry
+ }
+ p.mu.Unlock()
+
+ entry.mu.Lock()
+ defer entry.mu.Unlock()
+ if entry.token != nil && time.Now().Before(entry.expires) {
+ return entry.token, nil
+ }
+ token, lifetime, err := p.mint(ctx, subject, actor)
+ if err != nil {
+ return nil, err
+ }
+ reuse := min(lifetime-expirySkew, p.cacheTTL)
+ if reuse <= 0 {
+ return token, nil
+ }
+ entry.token, entry.expires = token, time.Now().Add(reuse)
+ return token, nil
+}
+
+// mint signs an assertion for subject and exchanges it at the node token
+// endpoint. The control plane verifies the assertion against this provider's
+// JWKS and resolves the subject's roles from the mesh policy.
+func (p *provider) mint(ctx context.Context, subject string, actor actorRef) ([]byte, time.Duration, error) {
+ assertion, err := p.signer.assertion(subject, map[string]string{"atespace": actor.atespace, "actorName": actor.name})
+ if err != nil {
+ return nil, 0, status.Errorf(codes.Internal, "signing assertion: %v", err)
+ }
+ form := url.Values{
+ "grant_type": {grantTypeTokenExchange},
+ "subject_token": {assertion},
+ "subject_token_type": {tokenTypeJWT},
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.node+"/oauth/token", strings.NewReader(form.Encode()))
+ if err != nil {
+ return nil, 0, status.Errorf(codes.Internal, "token request: %v", err)
+ }
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ resp, err := p.http.Do(req)
+ if err != nil {
+ return nil, 0, status.Errorf(codes.Unavailable, "agentmesh-node token endpoint: %v", err)
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+ if err != nil {
+ return nil, 0, status.Errorf(codes.Unavailable, "reading token response: %v", err)
+ }
+ var out struct {
+ AccessToken string `json:"access_token"`
+ ExpiresIn int64 `json:"expires_in"`
+ Error string `json:"error"`
+ Description string `json:"error_description"`
+ }
+ if err := json.Unmarshal(body, &out); err != nil {
+ return nil, 0, status.Errorf(codes.Unavailable, "token response is not JSON (HTTP %d)", resp.StatusCode)
+ }
+ if resp.StatusCode != http.StatusOK || out.AccessToken == "" {
+ slog.Warn("mesh refused the actor", slog.String("actor", subject), slog.Int("status", resp.StatusCode),
+ slog.String("error", out.Error), slog.String("description", out.Description))
+ return nil, 0, status.Errorf(codes.PermissionDenied, "mesh refused %s: %s %s", subject, out.Error, out.Description)
+ }
+ slog.Info("minted mesh credential", slog.String("actor", subject), slog.Int64("expires_in", out.ExpiresIn))
+ return []byte(out.AccessToken), time.Duration(out.ExpiresIn) * time.Second, nil
+}
+
+// signer holds the ES256 key the provider asserts actor identities with and
+// publishes it the way an OIDC issuer does.
+type signer struct {
+ issuer, audience string
+ kid string
+ key *ecdsa.PrivateKey
+ jwks, discovery []byte
+}
+
+func newSigner(issuer, audience string) (*signer, error) {
+ key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ if err != nil {
+ return nil, err
+ }
+ der, err := x509.MarshalPKIXPublicKey(&key.PublicKey)
+ if err != nil {
+ return nil, err
+ }
+ sum := sha256.Sum256(der)
+ s := &signer{issuer: issuer, audience: audience, kid: base64.RawURLEncoding.EncodeToString(sum[:]), key: key}
+ coord := func(b []byte) string {
+ padded := make([]byte, 32)
+ copy(padded[32-len(b):], b)
+ return base64.RawURLEncoding.EncodeToString(padded)
+ }
+ s.jwks, _ = json.Marshal(map[string]any{"keys": []map[string]string{{
+ "kty": "EC", "crv": "P-256", "alg": "ES256", "use": "sig", "kid": s.kid,
+ "x": coord(key.PublicKey.X.Bytes()), "y": coord(key.PublicKey.Y.Bytes()),
+ }}})
+ s.discovery, _ = json.Marshal(map[string]any{
+ "issuer": issuer,
+ "jwks_uri": issuer + "/keys",
+ "response_types_supported": []string{"id_token"},
+ "subject_types_supported": []string{"public"},
+ "id_token_signing_alg_values_supported": []string{"ES256"},
+ "claims_supported": []string{"iss", "sub", "aud", "exp", "iat", "nbf", "jti", "ate.dev"},
+ })
+ return s, nil
+}
+
+func (s *signer) assertion(subject string, actor map[string]string) (string, error) {
+ now := time.Now()
+ jti := make([]byte, 16)
+ if _, err := rand.Read(jti); err != nil {
+ return "", err
+ }
+ token := jwt.NewWithClaims(jwt.SigningMethodES256, jwt.MapClaims{
+ "iss": s.issuer,
+ "sub": subject,
+ "aud": []string{s.audience},
+ "iat": now.Unix(),
+ "nbf": now.Add(-time.Minute).Unix(),
+ "exp": now.Add(assertionLifetime).Unix(),
+ "jti": hex.EncodeToString(jti),
+ "ate.dev": actor,
+ })
+ token.Header["kid"] = s.kid
+ return token.SignedString(s.key)
+}
+
+func (s *signer) handler() http.Handler {
+ mux := http.NewServeMux()
+ serve := func(body []byte) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("Cache-Control", "no-store")
+ _, _ = w.Write(body)
+ }
+ }
+ mux.Handle("GET /.well-known/openid-configuration", serve(s.discovery))
+ mux.Handle("GET /keys", serve(s.jwks))
+ mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })
+ mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })
+ return mux
+}
+
+// serverCredentials builds the mutual TLS the gateway expects from a provider:
+// the pod's servicedns certificate on the server side, and on the client side
+// a certificate that chains to the pod-identity CA and carries the egress
+// gateway's SPIFFE ID. Both files rotate, so they are read on every handshake.
+func serverCredentials(bundlePath, caPath, expectedSAN string) (credentials.TransportCredentials, error) {
+ if _, err := loadBundle(bundlePath); err != nil {
+ return nil, err
+ }
+ if _, err := loadPool(caPath); err != nil {
+ return nil, err
+ }
+ cfg := &tls.Config{
+ MinVersion: tls.VersionTLS13,
+ GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {
+ pool, err := loadPool(caPath)
+ if err != nil {
+ return nil, err
+ }
+ return &tls.Config{
+ MinVersion: tls.VersionTLS13,
+ ClientAuth: tls.RequireAndVerifyClientCert,
+ ClientCAs: pool,
+ GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+ return loadBundle(bundlePath)
+ },
+ VerifyConnection: func(state tls.ConnectionState) error {
+ if len(state.PeerCertificates) == 0 {
+ return errors.New("client certificate is required")
+ }
+ for _, u := range state.PeerCertificates[0].URIs {
+ if u.String() == expectedSAN {
+ return nil
+ }
+ }
+ return fmt.Errorf("client certificate URI SANs %v do not include %q", state.PeerCertificates[0].URIs, expectedSAN)
+ },
+ }, nil
+ },
+ }
+ return credentials.NewTLS(cfg), nil
+}
+
+// loadBundle reads a Kubernetes pod certificate credential bundle: a PEM
+// private key followed by the certificate chain.
+func loadBundle(path string) (*tls.Certificate, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, fmt.Errorf("credential bundle: %w", err)
+ }
+ var cert tls.Certificate
+ for {
+ var block *pem.Block
+ block, data = pem.Decode(data)
+ if block == nil {
+ break
+ }
+ switch {
+ case block.Type == "CERTIFICATE":
+ cert.Certificate = append(cert.Certificate, block.Bytes)
+ case strings.HasSuffix(block.Type, "PRIVATE KEY") && cert.PrivateKey == nil:
+ key, err := parsePrivateKey(block)
+ if err != nil {
+ return nil, fmt.Errorf("credential bundle %s: %w", path, err)
+ }
+ cert.PrivateKey = key
+ }
+ }
+ if cert.PrivateKey == nil || len(cert.Certificate) == 0 {
+ return nil, fmt.Errorf("credential bundle %s has no key and certificate", path)
+ }
+ return &cert, nil
+}
+
+func parsePrivateKey(block *pem.Block) (any, error) {
+ if key, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil {
+ return key, nil
+ }
+ if key, err := x509.ParseECPrivateKey(block.Bytes); err == nil {
+ return key, nil
+ }
+ return x509.ParsePKCS1PrivateKey(block.Bytes)
+}
+
+func loadPool(path string) (*x509.CertPool, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, fmt.Errorf("client CA bundle: %w", err)
+ }
+ pool := x509.NewCertPool()
+ if !pool.AppendCertsFromPEM(data) {
+ return nil, fmt.Errorf("client CA bundle %s holds no certificates", path)
+ }
+ return pool, nil
+}
diff --git a/development/examples/substrate-actor/credential-provider/main_test.go b/development/examples/substrate-actor/credential-provider/main_test.go
new file mode 100644
index 00000000..338fbd59
--- /dev/null
+++ b/development/examples/substrate-actor/credential-provider/main_test.go
@@ -0,0 +1,198 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/agent-substrate/substrate/pkg/proto/credproviderpb"
+ "github.com/golang-jwt/jwt/v5"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+const demoActor = "spiffe://substrate-actor.local/actor/ate-demo-sandbox/demo"
+
+// fakeNode is the token endpoint: it verifies the assertion against the
+// provider's own JWKS, the way the control plane does, and answers with a
+// Biscuit for a bound subject or an OAuth error for anyone else.
+func fakeNode(t *testing.T, s *signer, bound string, calls *atomic.Int32) *httptest.Server {
+ t.Helper()
+ var keys jwt.VerificationKeySet
+ var set struct {
+ Keys []struct {
+ Kid, X, Y string
+ } `json:"keys"`
+ }
+ if err := json.Unmarshal(s.jwks, &set); err != nil || len(set.Keys) != 1 || set.Keys[0].Kid != s.kid {
+ t.Fatalf("jwks: %v %s", err, s.jwks)
+ }
+ keys.Keys = append(keys.Keys, &s.key.PublicKey)
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+ if r.URL.Path != "/oauth/token" || r.FormValue("grant_type") != grantTypeTokenExchange || r.FormValue("subject_token_type") != tokenTypeJWT {
+ http.Error(w, "unexpected request", http.StatusBadRequest)
+ return
+ }
+ claims := jwt.MapClaims{}
+ tok, err := jwt.ParseWithClaims(r.FormValue("subject_token"), claims, func(*jwt.Token) (any, error) { return keys, nil },
+ jwt.WithIssuer(s.issuer), jwt.WithAudience(s.audience), jwt.WithValidMethods([]string{"ES256"}))
+ if err != nil || !tok.Valid || tok.Header["kid"] != s.kid {
+ http.Error(w, "bad assertion: "+err.Error(), http.StatusUnauthorized)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ if sub, _ := claims["sub"].(string); sub != bound {
+ w.WriteHeader(http.StatusBadRequest)
+ _ = json.NewEncoder(w).Encode(map[string]string{"error": "invalid_grant", "error_description": "no role for " + sub})
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"access_token": "biscuit-for-" + bound, "token_type": "Bearer", "expires_in": 300})
+ }))
+}
+
+func newTestProvider(t *testing.T, node string, ttl time.Duration) (*provider, *signer) {
+ t.Helper()
+ s, err := newSigner("http://provider.test", "agentmesh")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return &provider{
+ name: "agentmesh", instance: "mesh", trustDomain: "substrate-actor.local", signer: s, node: node,
+ cacheTTL: ttl, http: &http.Client{Timeout: 5 * time.Second}, cache: map[string]*cached{},
+ }, s
+}
+
+func TestFetchSecretMintsAndCachesPerActor(t *testing.T) {
+ var calls atomic.Int32
+ p, s := newTestProvider(t, "", time.Minute)
+ node := fakeNode(t, s, demoActor, &calls)
+ defer node.Close()
+ p.node = node.URL
+
+ req := &credproviderpb.FetchSecretRequest{Uri: "ate-secret://agentmesh/mesh/biscuit", ActorSpiffeId: demoActor}
+ for i := range 3 {
+ resp, err := p.FetchSecret(context.Background(), req)
+ if err != nil {
+ t.Fatalf("call %d: %v", i, err)
+ }
+ if string(resp.GetOpaqueBytes()) != "biscuit-for-"+demoActor {
+ t.Fatalf("call %d: credential %q", i, resp.GetOpaqueBytes())
+ }
+ }
+ if got := calls.Load(); got != 1 {
+ t.Fatalf("expected one token exchange for three requests, got %d", got)
+ }
+
+ // A different actor is a different subject and a different exchange.
+ other := "spiffe://substrate-actor.local/actor/ate-demo-sandbox/rogue"
+ _, err := p.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: req.Uri, ActorSpiffeId: other})
+ if status.Code(err) != codes.PermissionDenied {
+ t.Fatalf("unbound actor: got %v, want PermissionDenied", err)
+ }
+ if got := calls.Load(); got != 2 {
+ t.Fatalf("expected the unbound actor to reach the node once, got %d calls", got)
+ }
+}
+
+func TestFetchSecretRefusesWhatItDoesNotServe(t *testing.T) {
+ var calls atomic.Int32
+ p, s := newTestProvider(t, "", time.Minute)
+ node := fakeNode(t, s, demoActor, &calls)
+ defer node.Close()
+ p.node = node.URL
+
+ for _, tc := range []struct {
+ name, uri, actor string
+ code codes.Code
+ }{
+ {"other provider", "ate-secret://k8s.io/agentmesh/secret/key", demoActor, codes.NotFound},
+ {"other instance", "ate-secret://agentmesh/other/biscuit", demoActor, codes.NotFound},
+ {"other tail", "ate-secret://agentmesh/mesh/token", demoActor, codes.NotFound},
+ {"query", "ate-secret://agentmesh/mesh/biscuit?scope=x", demoActor, codes.NotFound},
+ {"pod identity, not an actor", "ate-secret://agentmesh/mesh/biscuit", "spiffe://cluster.local/ns/ate-system/sa/atenet-egress", codes.PermissionDenied},
+ {"ateom identity, not an actor", "ate-secret://agentmesh/mesh/biscuit", "spiffe://substrate-actor.local/ateom-for-actor/a/b", codes.PermissionDenied},
+ {"empty actor", "ate-secret://agentmesh/mesh/biscuit", "", codes.PermissionDenied},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := p.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: tc.actor})
+ if status.Code(err) != tc.code {
+ t.Fatalf("got %v, want %v", err, tc.code)
+ }
+ })
+ }
+ if calls.Load() != 0 {
+ t.Fatalf("a refused request must not reach the node")
+ }
+}
+
+func TestCacheHonoursTTL(t *testing.T) {
+ var calls atomic.Int32
+ p, s := newTestProvider(t, "", 0)
+ node := fakeNode(t, s, demoActor, &calls)
+ defer node.Close()
+ p.node = node.URL
+ req := &credproviderpb.FetchSecretRequest{Uri: "ate-secret://agentmesh/mesh/biscuit", ActorSpiffeId: demoActor}
+ for range 2 {
+ if _, err := p.FetchSecret(context.Background(), req); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if calls.Load() != 2 {
+ t.Fatalf("cache-ttl 0 must mint on every request, got %d calls", calls.Load())
+ }
+}
+
+func TestDiscoveryDocument(t *testing.T) {
+ s, err := newSigner("http://provider.test", "agentmesh")
+ if err != nil {
+ t.Fatal(err)
+ }
+ srv := httptest.NewServer(s.handler())
+ defer srv.Close()
+ resp, err := http.Get(srv.URL + "/.well-known/openid-configuration")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ var doc struct {
+ Issuer string `json:"issuer"`
+ JWKSURI string `json:"jwks_uri"`
+ Algs []string `json:"id_token_signing_alg_values_supported"`
+ }
+ if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil {
+ t.Fatal(err)
+ }
+ if doc.Issuer != "http://provider.test" || doc.JWKSURI != "http://provider.test/keys" || len(doc.Algs) != 1 || doc.Algs[0] != "ES256" {
+ t.Fatalf("discovery document: %+v", doc)
+ }
+ keys, err := http.Get(srv.URL + "/keys")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer keys.Body.Close()
+ var set struct {
+ Keys []map[string]string `json:"keys"`
+ }
+ if err := json.NewDecoder(keys.Body).Decode(&set); err != nil || len(set.Keys) != 1 || set.Keys[0]["kid"] != s.kid || set.Keys[0]["crv"] != "P-256" {
+ t.Fatalf("jwks: %v %+v", err, set)
+ }
+}
diff --git a/development/examples/substrate-actor/egress-policy.yaml b/development/examples/substrate-actor/egress-policy.yaml
new file mode 100644
index 00000000..150f6fc7
--- /dev/null
+++ b/development/examples/substrate-actor/egress-policy.yaml
@@ -0,0 +1,21 @@
+# What the actor may reach, decided by Substrate's egress gateway. One rule:
+# plain HTTP to the edge node. On every request that carries an
+# X-Mesh-Authentication header, the gateway replaces its value with the
+# credential the Agent Mesh provider mints for this actor's SPIFFE ID.
+#
+# When Substrate serves OIDC discovery for actor JWTs (agent-substrate/substrate#1756),
+# the same policy works with the provider removed:
+# replaceHeaders:
+# - header: X-Mesh-Authentication
+# prefix: "Bearer "
+# actorJwt: {audiences: ["agentmesh"], expirationSeconds: 600}
+rules:
+- http:
+ hostnames: ["edge-node.agentmesh.svc.cluster.local"]
+ ports:
+ numbers: [8080]
+ effects:
+ replaceHeaders:
+ - header: X-Mesh-Authentication
+ prefix: "Bearer "
+ credentialUri: ate-secret://agentmesh/mesh/biscuit
diff --git a/development/examples/substrate-actor/manifests/credential-provider.yaml.tmpl b/development/examples/substrate-actor/manifests/credential-provider.yaml.tmpl
new file mode 100644
index 00000000..56590d15
--- /dev/null
+++ b/development/examples/substrate-actor/manifests/credential-provider.yaml.tmpl
@@ -0,0 +1,81 @@
+# The Agent Mesh credential provider for Substrate's egress gateway. It is a
+# CredentialProvider (pkg/proto/credproviderpb) like the bundled Kubernetes
+# Secrets one, deployed the same way: a servicedns pod certificate to serve
+# with, and the pod-identity trust bundle to verify the gateway's client
+# certificate against. `ate-setup` points the gateway at it with
+# --credential-provider '{"name":"agentmesh","address":"credential-provider.agentmesh.svc:50051"}'
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: credential-provider
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: credential-provider}
+ template:
+ metadata:
+ labels: {app: credential-provider}
+ spec:
+ containers:
+ - name: credential-provider
+ image: ${CREDENTIAL_PROVIDER_IMAGE}
+ args:
+ - --listen-address=:50051
+ - --http-address=:8080
+ - --server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem
+ - --client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem
+ - --injector-identity=spiffe://cluster.local/ns/ate-system/sa/atenet-egress
+ - --provider-name=agentmesh
+ - --node-url=http://edge-node.agentmesh.svc.cluster.local:8080
+ # Must equal the control plane's --workload-issuer, verbatim.
+ - --issuer=http://credential-provider.agentmesh.svc.cluster.local:8080
+ - --audience=agentmesh
+ - --cache-ttl=60s
+ ports:
+ - {name: grpc, containerPort: 50051}
+ - {name: http, containerPort: 8080}
+ readinessProbe:
+ httpGet: {path: /readyz, port: http}
+ volumeMounts:
+ - {name: servicedns, mountPath: /run/servicedns.podcert.ate.dev, readOnly: true}
+ - {name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true}
+ volumes:
+ - name: servicedns
+ projected:
+ sources:
+ - podCertificate:
+ signerName: servicedns.podcert.ate.dev/identity
+ keyType: ECDSAP256
+ credentialBundlePath: credential-bundle.pem
+ - name: podidentity
+ projected:
+ sources:
+ - clusterTrustBundle:
+ signerName: podidentity.podcert.ate.dev/identity
+ labelSelector:
+ matchLabels:
+ podcert.ate.dev/canarying: live
+ path: trust-bundle.pem
+---
+# Only the egress gateway may reach the gRPC port; the HTTP port serves the
+# public discovery document the control plane reads.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: credential-provider
+ namespace: agentmesh
+spec:
+ podSelector:
+ matchLabels: {app: credential-provider}
+ policyTypes: [Ingress]
+ ingress:
+ - from:
+ - namespaceSelector:
+ matchLabels: {kubernetes.io/metadata.name: ate-system}
+ podSelector:
+ matchLabels: {app: atenet-egress}
+ ports:
+ - {protocol: TCP, port: 50051}
+ - ports:
+ - {protocol: TCP, port: 8080}
diff --git a/development/examples/substrate-actor/manifests/mesh.yaml.tmpl b/development/examples/substrate-actor/manifests/mesh.yaml.tmpl
new file mode 100644
index 00000000..584e1628
--- /dev/null
+++ b/development/examples/substrate-actor/manifests/mesh.yaml.tmpl
@@ -0,0 +1,205 @@
+# The Agent Mesh beside the Substrate cluster, in one namespace: agentmesh-one
+# (control plane, router, console), an edge node that is the actors' door to
+# the mesh and the provider's token endpoint, an office node fronting a
+# calculator, and the credential provider the Substrate egress gateway calls.
+#
+# Everything the actors reach is plain HTTP on cluster DNS. The egress gateway
+# verifies the actor's certificate and injects the mesh credential; the actor
+# itself holds nothing.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: mesh-config
+ namespace: agentmesh
+data:
+ policy.json: |
+ {
+ "roles": [
+ {"name": "mesh:role:node", "allowed_services": ["*"], "allowed_targets": ["*"], "allowed_labels": ["*"]},
+ {"name": "mesh:role:router", "allowed_services": ["*"], "allowed_targets": ["*"]},
+ {"name": "actor",
+ "allowed_services": ["system://mesh.catalog", "mcp://calc"],
+ "allowed_targets": ["*"]}
+ ],
+ "bindings": [
+ {"role": "actor", "members": ["user:spiffe://substrate-actor.local/actor/ate-demo-sandbox/demo"]}
+ ]
+ }
+ office.yaml: |
+ version: "v1alpha1"
+ labels:
+ site: office
+ services:
+ - type: mcp
+ name: calc
+ description: "calculator MCP server in the office"
+ target_url: "http://calc-mcp.agentmesh.svc.cluster.local:7777/mcp"
+ - type: mcp
+ name: payroll
+ description: "an MCP server the actor role may not call"
+ target_url: "http://calc-mcp.agentmesh.svc.cluster.local:7777/mcp"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: agentmesh-one
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: agentmesh-one}
+ template:
+ metadata:
+ labels: {app: agentmesh-one}
+ spec:
+ containers:
+ - name: agentmesh-one
+ image: ${AGENTMESH_ONE_IMAGE}
+ args:
+ - --bind-address=0.0.0.0
+ - --port=8080
+ - --data-dir=/data
+ - --policy-file=/etc/agentmesh/policy.json
+ - --token-path=/run/agentmesh/join-token
+ - --admin-token-path=/run/agentmesh/admin-token
+ # The credential provider asserts actor identities; the mesh trusts
+ # it as a workload identity issuer and binds roles to the subject.
+ - --workload-issuer=http://credential-provider.agentmesh.svc.cluster.local:8080
+ - --allowed-audiences=agentmesh
+ - --enroll-qr=false
+ ports:
+ - {name: http, containerPort: 8080}
+ volumeMounts:
+ - {name: config, mountPath: /etc/agentmesh, readOnly: true}
+ - {name: tokens, mountPath: /run/agentmesh, readOnly: true}
+ - {name: data, mountPath: /data}
+ volumes:
+ - name: config
+ configMap: {name: mesh-config}
+ - name: tokens
+ secret: {secretName: agentmesh-tokens, defaultMode: 0444}
+ - name: data
+ emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: agentmesh-one
+ namespace: agentmesh
+spec:
+ selector: {app: agentmesh-one}
+ ports:
+ - {name: http, port: 8080, targetPort: http}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: edge-node
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: edge-node}
+ template:
+ metadata:
+ labels: {app: edge-node}
+ spec:
+ containers:
+ - name: agentmesh-node
+ image: ${AGENTMESH_NODE_IMAGE}
+ args:
+ - run
+ - --control-plane=http://agentmesh-one.agentmesh.svc.cluster.local:8080
+ - --insecure-control-plane
+ - --bootstrap-token-path=/run/agentmesh/join-token
+ - --api-token-path=/run/agentmesh/api-token
+ - --data-dir=/data
+ - --bind-addr=0.0.0.0:8080
+ - --control-plane-sync-interval=30s
+ ports:
+ - {name: http, containerPort: 8080}
+ volumeMounts:
+ - {name: tokens, mountPath: /run/agentmesh, readOnly: true}
+ - {name: data, mountPath: /data}
+ volumes:
+ - name: tokens
+ secret: {secretName: agentmesh-tokens, defaultMode: 0444}
+ - name: data
+ emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: edge-node
+ namespace: agentmesh
+spec:
+ selector: {app: edge-node}
+ ports:
+ - {name: http, port: 8080, targetPort: http}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: office-node
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: office-node}
+ template:
+ metadata:
+ labels: {app: office-node}
+ spec:
+ containers:
+ - name: agentmesh-node
+ image: ${AGENTMESH_NODE_IMAGE}
+ args:
+ - run
+ - --control-plane=http://agentmesh-one.agentmesh.svc.cluster.local:8080
+ - --insecure-control-plane
+ - --bootstrap-token-path=/run/agentmesh/join-token
+ - --api-token-path=/run/agentmesh/api-token
+ - --data-dir=/data
+ - --bind-addr=
+ - --config=/etc/agentmesh/office.yaml
+ - --control-plane-sync-interval=30s
+ volumeMounts:
+ - {name: config, mountPath: /etc/agentmesh, readOnly: true}
+ - {name: tokens, mountPath: /run/agentmesh, readOnly: true}
+ - {name: data, mountPath: /data}
+ volumes:
+ - name: config
+ configMap: {name: mesh-config}
+ - name: tokens
+ secret: {secretName: agentmesh-tokens, defaultMode: 0444}
+ - name: data
+ emptyDir: {}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: calc-mcp
+ namespace: agentmesh
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {app: calc-mcp}
+ template:
+ metadata:
+ labels: {app: calc-mcp}
+ spec:
+ containers:
+ - name: calc-mcp
+ image: ${CALC_MCP_IMAGE}
+ ports:
+ - {name: http, containerPort: 7777}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: calc-mcp
+ namespace: agentmesh
+spec:
+ selector: {app: calc-mcp}
+ ports:
+ - {name: http, port: 7777, targetPort: http}
diff --git a/development/examples/substrate-actor/manifests/namespace.yaml b/development/examples/substrate-actor/manifests/namespace.yaml
new file mode 100644
index 00000000..3fa95127
--- /dev/null
+++ b/development/examples/substrate-actor/manifests/namespace.yaml
@@ -0,0 +1,17 @@
+# Created before Substrate so that the egress gateway's provider address
+# resolves from the start; the provider itself arrives with `make up`.
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: agentmesh
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: credential-provider
+ namespace: agentmesh
+spec:
+ selector: {app: credential-provider}
+ ports:
+ - {name: grpc, port: 50051, targetPort: grpc}
+ - {name: http, port: 8080, targetPort: http}
diff --git a/development/examples/substrate-actor/mcp.py b/development/examples/substrate-actor/mcp.py
new file mode 100755
index 00000000..be615c0d
--- /dev/null
+++ b/development/examples/substrate-actor/mcp.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+# Calls the mesh from inside a Substrate actor, the way an agent would: plain
+# HTTP to the edge node with a placeholder credential. The Substrate egress
+# gateway replaces the placeholder with the actor's mesh credential.
+#
+# The actor is Substrate's sandbox demo (Alpine, POST /process runs a command
+# in the sandbox). This script reaches it through atenet-router on
+# 127.0.0.1:${ROUTER_PORT:-8000} with the ate-target-actor header.
+#
+# mcp.py tools ACTOR [SERVICE] tools the mesh offers this actor
+# mcp.py add ACTOR SERVICE A B call mcp://SERVICE/add through the mesh
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+
+ROUTER = "http://127.0.0.1:" + os.environ.get("ROUTER_PORT", "8000")
+ATESPACE = os.environ.get("ATESPACE", "ate-demo-sandbox")
+NODE = "http://edge-node.agentmesh.svc.cluster.local:8080/mcp"
+
+
+def in_actor(actor, command, timeout="60s"):
+ body = json.dumps({"command": command, "timeout": timeout}).encode()
+ req = urllib.request.Request(ROUTER + "/process", data=body, method="POST", headers={
+ "Content-Type": "application/json",
+ "ate-target-actor": f"{ATESPACE}/{actor}",
+ })
+ try:
+ with urllib.request.urlopen(req, timeout=120) as resp:
+ out = json.load(resp)
+ except urllib.error.HTTPError as e:
+ sys.exit(f"router answered HTTP {e.code} for actor {ATESPACE}/{actor}: {e.read().decode()[:300]}")
+ if out.get("error"):
+ sys.exit(f"actor could not run the command: {out['error']} {out.get('stderr', '')}")
+ return out["stdout"], out.get("exitCode", 0)
+
+
+def rpc(actor, tool, arguments):
+ body = json.dumps({
+ "jsonrpc": "2.0", "id": 1, "method": "tools/call",
+ "params": {"name": tool, "arguments": arguments},
+ })
+ # BusyBox wget speaks HTTP/1.1 well enough for one request; -S puts the
+ # status line on stderr, which /process returns separately.
+ out, code = in_actor(actor, [
+ "wget", "-q", "-O", "-", "-T", "60",
+ "--header=Content-Type: application/json",
+ "--header=Accept: application/json, text/event-stream",
+ "--header=X-Mesh-Authentication: Bearer placeholder",
+ f"--post-data={body}", NODE,
+ ])
+ payload = out
+ if out.lstrip().startswith("event:") or "\ndata: " in out:
+ payload = "".join(line[6:] for line in out.splitlines() if line.startswith("data: "))
+ try:
+ reply = json.loads(payload)
+ except ValueError:
+ sys.exit(f"{tool}: wget exit {code}: {out.strip()[:300] or '(no body; the egress gateway refused the request)'}")
+ if "error" in reply:
+ error = reply["error"]
+ sys.exit(f"{tool}: {error.get('message', error) if isinstance(error, dict) else error}")
+ result = reply["result"]
+ text = "\n".join(c.get("text", "") for c in result.get("content", []) if c.get("type") == "text")
+ if result.get("isError"):
+ return None, text
+ try:
+ return json.loads(text), text
+ except ValueError:
+ return text, text
+
+
+def tools(actor, service=None):
+ rows, text = rpc(actor, "find_remote_tools", {"service_name": service} if service else {})
+ if rows is None:
+ sys.exit(f"find_remote_tools refused: {text}")
+ if isinstance(rows, dict):
+ rows = rows.get("tools") or rows.get("results") or [rows]
+ for row in rows:
+ if "error" in row:
+ print(f" {row.get('peer_id', '?')[:16]}… {row['error']}")
+ else:
+ print(f" {row['peer_id'][:16]}… {row['tool_name']:<24} {row.get('description', '')}")
+ return rows
+
+
+def add(actor, service, a, b):
+ rows = tools(actor, service)
+ peers = {r["peer_id"] for r in rows if r.get("tool_name") == f"mcp://{service}/add"}
+ if not peers:
+ print(f" (no tool of mcp://{service} is visible to this actor; calling the office node directly)")
+ peers = {r["peer_id"] for r in rpc(actor, "discover_remote_services", {"type": "mcp"})[0] or []}
+ if not peers:
+ print(f"mcp://{service}/add({a}, {b}) -> refused: the mesh shows this actor no MCP provider at all (no role binds its identity)")
+ sys.exit(3)
+ peer = sorted(peers)[0]
+ result, text = rpc(actor, "call_remote_tool", {
+ "peer_id": peer, "tool_name": f"mcp://{service}/add",
+ "arguments": {"a": float(a), "b": float(b)},
+ })
+ if result is None:
+ print(f"mcp://{service}/add({a}, {b}) -> refused: {text}")
+ sys.exit(3)
+ print(f"mcp://{service}/add({a}, {b}) = {text}")
+
+
+if __name__ == "__main__":
+ cmd, *rest = sys.argv[1:] or ["help"]
+ if cmd == "tools" and 1 <= len(rest) <= 2:
+ tools(*rest)
+ elif cmd == "add" and len(rest) == 4:
+ add(*rest)
+ else:
+ sys.exit("usage: mcp.py tools ACTOR [SERVICE] | add ACTOR SERVICE A B")
diff --git a/site/content/docs/guides/_index.md b/site/content/docs/guides/_index.md
index f657265f..eb644412 100644
--- a/site/content/docs/guides/_index.md
+++ b/site/content/docs/guides/_index.md
@@ -9,6 +9,6 @@ aliases:
Step-by-step pages for common tasks: publish a service, connect an agent
client, enroll machines that cannot log in, put an agent in an OpenShell
-sandbox on the mesh, and deploy a control plane on Kubernetes, Cloud Run,
-SkyPilot or a GitHub codespace. Each page assumes that you have read the
-[quick start](../getting-started/quickstart/).
+sandbox or a Substrate actor on the mesh, and deploy a control plane on
+Kubernetes, Cloud Run, SkyPilot or a GitHub codespace. Each page assumes
+that you have read the [quick start](../getting-started/quickstart/).
diff --git a/site/content/docs/guides/openshell.md b/site/content/docs/guides/openshell.md
index cb1d5694..9aec11d0 100644
--- a/site/content/docs/guides/openshell.md
+++ b/site/content/docs/guides/openshell.md
@@ -1,7 +1,7 @@
---
title: "OpenShell sandboxes"
linkTitle: "OpenShell sandboxes"
-weight: 4
+weight: 5
---
This guide puts an agent that runs in an [NVIDIA OpenShell](https://github.com/NVIDIA/OpenShell)
diff --git a/site/content/docs/guides/substrate.md b/site/content/docs/guides/substrate.md
new file mode 100644
index 00000000..ea599e49
--- /dev/null
+++ b/site/content/docs/guides/substrate.md
@@ -0,0 +1,287 @@
+---
+title: "Substrate actors"
+linkTitle: "Substrate actors"
+weight: 5
+---
+
+This guide puts an [Agent Substrate](https://github.com/agent-substrate/substrate)
+actor on the mesh under its own identity. Substrate runs agents as actors in
+gVisor sandboxes and sends all of their outbound traffic through an egress
+gateway that authenticates each actor by certificate and enforces a
+per-actor `EgressPolicy`. That gateway can also replace a request header
+with a credential it fetches from a credential provider. This guide adds an
+Agent Mesh credential provider: for each actor, it asserts the actor's
+SPIFFE ID to the mesh and hands the gateway the credential the mesh mints
+for it. The actor holds nothing and sends a placeholder; the mesh policy
+decides what each actor identity may call, and revoking an actor is a policy
+change.
+
+Nothing here modifies Substrate. The provider implements Substrate's
+`CredentialProvider` gRPC contract and is registered with the installer flag
+Substrate already has for that.
+
+Everything runs on one kind cluster:
+[`development/examples/substrate-actor/`](https://github.com/google/agentmesh/tree/main/development/examples/substrate-actor)
+in the repository has the provider, the manifests and a Makefile for every
+step below.
+
+## How the pieces fit
+
+```mermaid
+flowchart LR
+ subgraph Worker["Substrate worker pod"]
+ Actor["actor (gVisor)"]
+ Tunnel["atunnel<br/>actor X.509 cert"]
+ end
+ subgraph System["ate-system"]
+ Egress["atenet-egress<br/>verifies cert, applies EgressPolicy"]
+ end
+ subgraph Mesh["agentmesh namespace"]
+ Provider["credential-provider<br/>FetchSecret(uri, actor SPIFFE ID)"]
+ Edge["edge agentmesh-node"]
+ One["agentmesh-one"]
+ Office["office agentmesh-node<br/>mcp://calc, mcp://payroll"]
+ end
+ Actor -- "POST /mcp<br/>X-Mesh-Authentication: placeholder" --> Tunnel
+ Tunnel -- "mTLS CONNECT" --> Egress
+ Egress -- "ate-secret://agentmesh/mesh/biscuit" --> Provider
+ Provider -- "assertion for spiffe://…/actor/<atespace>/<name>" --> Edge
+ Edge -- "/token/exchange" --> One
+ Egress -- "X-Mesh-Authentication: Bearer biscuit" --> Edge
+ Edge -- "mesh stream, caller's biscuit" --> Office
+```
+
+1. The actor sends a plain MCP request to
+ `edge-node.agentmesh.svc.cluster.local:8080` with
+ `X-Mesh-Authentication: Bearer placeholder`. Substrate's `atunnel` carries
+ it to the egress gateway over mTLS with the actor's certificate.
+2. The gateway verifies the certificate, finds the matching `EgressPolicy`
+ rule, and because the request carries the header the rule names, asks the
+ credential provider for `ate-secret://agentmesh/mesh/biscuit` on behalf of
+ `spiffe://substrate-actor.local/actor/<atespace>/<name>`. The gateway
+ authenticates to the provider with its pod certificate; the provider
+ accepts no other caller.
+3. The provider signs a short-lived assertion whose subject is that SPIFFE
+ ID and exchanges it at the edge node's token endpoint (RFC 8693). The
+ control plane verifies the assertion against the provider's published
+ keys, since the provider is configured as a workload identity issuer, and
+ mints a Biscuit with the roles bound to that subject. The provider caches
+ it per actor.
+4. The gateway replaces the header value with `Bearer <biscuit>` and forwards
+ the request. The edge node passes the request into the mesh under that
+ identity, and the office node enforces the policy on its side.
+
+The provider is the attester here: the mesh trusts it to name actors
+truthfully, and it names only what the egress gateway, which verified the
+certificate, told it. Once Substrate serves OpenID discovery for the actor
+JWTs it already mints
+([agent-substrate/substrate#1756](https://github.com/agent-substrate/substrate/issues/1756)),
+the provider becomes unnecessary: the gateway's `actorJwt` injection puts a
+Substrate-signed JWT on the header, and `agentmesh-one` trusts Substrate's
+issuer directly. The last section shows that change.
+
+## Before you start
+
+- Docker, kind, ko, kubectl, Go and python3.
+- A checkout of Agent Substrate; the example installs it from source with
+ Substrate's own scripts.
+- `agentmesh-one` and `agentmesh-node` images newer than v0.1.0-rc.9. The
+ manifests use the `latest` images; set `AGENTMESH_VERSION` to pin a release.
+
+```bash
+cd development/examples/substrate-actor
+```
+
+## 1. Substrate, with the gateway pointed at the provider
+
+```bash
+make cluster SUBSTRATE_DIR=~/src/substrate
+```
+
+creates the kind cluster with Substrate's `hack/create-kind-cluster.sh` and
+installs Substrate with
+
+```bash
+./hack/install-ate-kind.sh --deploy-ate-system --deploy-demo-sandbox \
+ --credential-provider='{"name":"agentmesh","address":"credential-provider.agentmesh.svc:50051"}'
+```
+
+`--credential-provider` is the installer's existing option for a provider you
+deploy yourself. `name` becomes the provider class of the credential URIs
+(`ate-secret://agentmesh/...`) and `address` is where the egress gateway dials
+it. The gateway presents its pod-identity certificate and verifies the
+provider's serving certificate against the cluster's servicedns trust bundle,
+so the provider's Service must exist before Substrate starts; the Makefile
+creates the `agentmesh` namespace and the Service first.
+
+`--deploy-demo-sandbox` installs Substrate's sandbox demo: an Alpine actor
+whose `POST /process` runs a command inside the sandbox. It stands in for an
+agent; any image that speaks HTTP takes its place.
+
+## 2. The provider and the mesh
+
+```bash
+make up
+```
+
+builds the provider with `ko`, loads the calculator image into kind, and
+applies `manifests/`. The provider runs with the same two projected volumes
+as Substrate's bundled Kubernetes Secrets provider:
+
+```yaml
+args:
+- --server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem
+- --client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem
+- --injector-identity=spiffe://cluster.local/ns/ate-system/sa/atenet-egress
+- --node-url=http://edge-node.agentmesh.svc.cluster.local:8080
+- --issuer=http://credential-provider.agentmesh.svc.cluster.local:8080
+- --audience=agentmesh
+- --cache-ttl=60s
+```
+
+`--issuer` is the `iss` of the assertions the provider signs; it serves the
+matching discovery document and JWKS on its HTTP port. `agentmesh-one` names
+the same URL:
+
+```text
+--workload-issuer=http://credential-provider.agentmesh.svc.cluster.local:8080
+--allowed-audiences=agentmesh
+```
+
+The mesh policy in `manifests/mesh.yaml.tmpl` binds the actor's identity to a
+role:
+
+```json
+{
+ "roles": [
+ { "name": "actor",
+ "allowed_services": ["system://mesh.catalog", "mcp://calc"],
+ "allowed_targets": ["*"] }
+ ],
+ "bindings": [
+ { "role": "actor",
+ "members": ["user:spiffe://substrate-actor.local/actor/ate-demo-sandbox/demo"] }
+ ]
+}
+```
+
+The subject is the SPIFFE ID Substrate gives the actor: trust domain
+`substrate-actor.local`, then `actor/<atespace>/<name>`. The office node
+publishes the calculator twice, as `mcp://calc` and `mcp://payroll`, so the
+policy has something to refuse.
+
+## 3. The actor's egress policy
+
+```bash
+make actor NAME=demo
+```
+
+creates the actor from the demo template and gives it this `EgressPolicy`:
+
+```yaml
+rules:
+- http:
+ hostnames: ["edge-node.agentmesh.svc.cluster.local"]
+ ports:
+ numbers: [8080]
+ effects:
+ replaceHeaders:
+ - header: X-Mesh-Authentication
+ prefix: "Bearer "
+ credentialUri: ate-secret://agentmesh/mesh/biscuit
+```
+
+One host, one port, and the rule the gateway applies to it. A request without
+the header is forwarded unchanged, so the actor opts in by sending the
+placeholder; a request to any other destination is refused by the gateway
+before it leaves the cluster.
+
+## 4. Calling the mesh from the actor
+
+```bash
+make call NAME=demo
+```
+
+```text
+== tools the mesh offers actor demo
+ 12D3KooWMaj9ere9… mcp://calc/add Return a + b.
+ 12D3KooWMaj9ere9… mcp://calc/multiply Return a * b.
+== mcp://calc, allowed for the actor role
+mcp://calc/add(2, 3) = 5.0
+== mcp://payroll, not in the actor role
+ (no tool of mcp://payroll is visible to this actor; calling the office node directly)
+mcp://payroll/add(2, 3) -> refused: auth rejected by 12D3KooWMaj9ere9…: biscuit: denied by no matching policies
+```
+
+Each line is a `wget` run inside the actor through Substrate's ingress
+router (`ate-target-actor: ate-demo-sandbox/demo`). The egress gateway's
+access log shows the actor's certificate and the request it allowed; the
+provider logs one mint per actor and serves the rest from its cache; the
+control plane logs the exchange that produced the credential:
+
+```text
+Border Crossing {"endpoint": "/token/exchange", "decision": "allow",
+ "subject": "spiffe://substrate-actor.local/actor/ate-demo-sandbox/demo", "roles": ["actor"]}
+```
+
+## 5. Same template, same policy, different identity
+
+```bash
+make rogue
+```
+
+creates a second actor, `rogue`, from the same template with the same egress
+policy. The gateway fetches a credential for
+`spiffe://substrate-actor.local/actor/ate-demo-sandbox/rogue`, the mesh mints
+one with no roles, and the office node shows it nothing:
+
+```text
+== tools the mesh offers actor rogue
+== mcp://calc, allowed for the actor role
+mcp://calc/add(2, 3) -> refused: the mesh shows this actor no MCP provider at all (no role binds its identity)
+```
+
+Add `user:spiffe://substrate-actor.local/actor/ate-demo-sandbox/rogue` to the
+`actor` role's members in `manifests/mesh.yaml.tmpl`, apply the ConfigMap,
+and push the policy:
+
+```bash
+make policy
+make call NAME=rogue
+```
+
+The running actor is allowed on its next request; an actor that already held
+a credential changes within `--cache-ttl`. Removing the member revokes it the
+same way. Neither step touches the actor, its template, or Substrate.
+
+## When Substrate serves its own issuer
+
+Substrate's egress gateway can already mint and inject a JWT for the calling
+actor (`actorJwt` in `replaceHeaders`), signed by the control plane with
+`iss` set to Substrate's `--actor-jwt-issuer`. What is missing today is a
+server for that issuer's discovery document and keys; it is tracked in
+[agent-substrate/substrate#1756](https://github.com/agent-substrate/substrate/issues/1756).
+When it lands, the provider goes away and two things change:
+
+```yaml
+# EgressPolicy
+ replaceHeaders:
+ - header: X-Mesh-Authentication
+ prefix: "Bearer "
+ actorJwt: {audiences: ["agentmesh"], expirationSeconds: 600}
+```
+
+```text
+# agentmesh-one
+--workload-issuer=https://idp.ate-system.svc
+```
+
+The mesh policy keeps the same subjects. The edge node exchanges the actor
+JWT on each request the way it exchanges any platform JWT.
+
+## Cleaning up
+
+```bash
+make down # the actors and the agentmesh namespace
+make clean # the kind cluster
+```
diff --git a/site/content/docs/preview/sandboxed-agents.md b/site/content/docs/preview/sandboxed-agents.md
index 428253b0..1a0d40ab 100644
--- a/site/content/docs/preview/sandboxed-agents.md
+++ b/site/content/docs/preview/sandboxed-agents.md
@@ -85,7 +85,9 @@ The sandbox still holds nothing. What changes is that the mesh, not the
orchestrator, decides what each sandbox identity may call, and revoking the
sandbox's role takes effect without touching the proxy. The
[OpenShell sandboxes guide](../../guides/openshell/) runs this end to end on
-one machine with SPIRE.
+one machine with SPIRE, and the [Substrate actors guide](../../guides/substrate/)
+does the same on a kind cluster through Substrate's credential provider
+contract.
## Blueprint 2: Docker Sandbox (`docker sbx`) & Kubernetes `agent-sandbox`