Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Command credential-provider is an Agent Substrate credential provider that
// gives every actor its own Agent Mesh credential.
//
// Substrate's egress gateway calls FetchSecret with the URI named in the
// actor's EgressPolicy and the actor's SPIFFE ID, which the gateway verified
// from the actor's client certificate. For ate-secret://agentmesh/mesh/biscuit
// this provider signs a short-lived assertion that names that SPIFFE ID as the
// subject, exchanges it at an agentmesh-node token endpoint (RFC 8693), and
// returns the Biscuit the mesh minted. The control plane trusts this provider
// as a workload identity issuer and binds roles to the actor's SPIFFE ID, so
// the mesh policy, not the provider, decides what each actor may call.
package main
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/hex"
"encoding/json"
"encoding/pem"
"errors"
"flag"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"github.com/agent-substrate/substrate/pkg/proto/credproviderpb"
"github.com/golang-jwt/jwt/v5"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/status"
)
const (
grantTypeTokenExchange = "urn:ietf:params:oauth:grant-type:token-exchange"
tokenTypeJWT = "urn:ietf:params:oauth:token-type:jwt"
// Substrate credential URIs are ate-secret://<provider-class>/<provider-name>/<tail>.
credentialTail = "biscuit"
assertionLifetime = 10 * time.Minute
expirySkew = 30 * time.Second
)
var (
listenAddr = flag.String("listen-address", ":50051", "gRPC listen address for the Substrate CredentialProvider service")
httpAddr = flag.String("http-address", ":8080", "HTTP listen address for OIDC discovery, the JWKS and health")
serverBundle = flag.String("server-cred-bundle", "", "PEM private key and certificate chain this provider serves TLS with (required)")
clientCAFile = flag.String("client-ca-file", "", "CA bundle the egress gateway's client certificate must chain to (required)")
injectorIdentity = flag.String("injector-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-egress", "URI SAN the egress gateway's client certificate must carry")
providerName = flag.String("provider-name", "agentmesh", "provider class of the ate-secret:// URIs this provider serves")
instanceName = flag.String("instance-name", "mesh", "provider name segment of the ate-secret:// URIs this provider serves")
actorTrustDomain = flag.String("actor-trust-domain", "substrate-actor.local", "SPIFFE trust domain of actor identities")
nodeURL = flag.String("node-url", "", "agentmesh-node API whose /oauth/token mints the credential (required)")
issuer = flag.String("issuer", "", "iss of the assertions this provider signs; the control plane fetches <issuer>/.well-known/openid-configuration (required)")
audience = flag.String("audience", "agentmesh", "aud of the assertions; one of the control plane's --allowed-audiences")
cacheTTL = flag.Duration("cache-ttl", time.Minute, "longest a minted credential is reused; also how long a mesh policy change takes to reach an actor")
signingKeyPath = flag.String("signing-key-path", "", "PEM file (PKCS#8 or SEC 1) with the P-256 key the assertions are signed with; generated per run when empty, in which case the issuer must be this provider's own HTTP address")
jwksURI = flag.String("jwks-uri", "", "jwks_uri the discovery document names; defaults to <issuer>/keys. Set it when the documents are published elsewhere (a bucket) and the issuer is not this process")
printDiscovery = flag.Bool("print-discovery", false, "write the discovery document and the JWKS for --signing-key-path to --print-discovery-dir and exit, for publishing them where the control plane reads the issuer")
printDiscoveryTo = flag.String("print-discovery-dir", ".", "directory --print-discovery writes .well-known/openid-configuration and keys into")
)
func main() {
flag.Parse()
logger := slog.New(slog.NewJSONHandler(os.Stderr, nil))
slog.SetDefault(logger)
if err := run(context.Background()); err != nil {
logger.Error("exiting", slog.Any("err", err))
os.Exit(1)
}
}
func run(ctx context.Context) error {
if *printDiscovery {
if *issuer == "" || *signingKeyPath == "" {
return errors.New("--print-discovery needs --issuer and --signing-key-path")
}
signer, err := newSigner(*issuer, *audience, *signingKeyPath, *jwksURI)
if err != nil {
return err
}
return signer.writeDiscovery(*printDiscoveryTo)
}
for name, value := range map[string]string{
"--server-cred-bundle": *serverBundle, "--client-ca-file": *clientCAFile,
"--node-url": *nodeURL, "--issuer": *issuer,
} {
if value == "" {
return fmt.Errorf("%s is required", name)
}
}
ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM)
defer stop()
signer, err := newSigner(*issuer, *audience, *signingKeyPath, *jwksURI)
if err != nil {
return err
}
provider := &provider{
name: *providerName,
instance: *instanceName,
trustDomain: *actorTrustDomain,
signer: signer,
node: strings.TrimRight(*nodeURL, "/"),
cacheTTL: *cacheTTL,
http: &http.Client{Timeout: 15 * time.Second},
cache: map[string]*cached{},
}
creds, err := serverCredentials(*serverBundle, *clientCAFile, *injectorIdentity)
if err != nil {
return err
}
grpcSrv := grpc.NewServer(grpc.Creds(creds))
credproviderpb.RegisterCredentialProviderServer(grpcSrv, provider)
lis, err := (&net.ListenConfig{}).Listen(ctx, "tcp", *listenAddr)
if err != nil {
return fmt.Errorf("listen %s: %w", *listenAddr, err)
}
httpSrv := &http.Server{Addr: *httpAddr, Handler: signer.handler(provider.cacheSize), ReadHeaderTimeout: 5 * time.Second}
errs := make(chan error, 2)
go func() { errs <- grpcSrv.Serve(lis) }()
go func() { errs <- httpSrv.ListenAndServe() }()
slog.Info("serving", slog.String("grpc", *listenAddr), slog.String("http", *httpAddr),
slog.String("issuer", *issuer), slog.String("node", provider.node), slog.String("uri", provider.uri()))
select {
case <-ctx.Done():
case err := <-errs:
if err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
}
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
grpcSrv.GracefulStop()
return httpSrv.Shutdown(shutdownCtx)
}
// provider answers FetchSecret for one URI with a per-actor mesh credential.
type provider struct {
credproviderpb.UnimplementedCredentialProviderServer
name string
instance string
trustDomain string
signer *signer
node string
cacheTTL time.Duration
http *http.Client
mu sync.Mutex
cache map[string]*cached
}
type cached struct {
mu sync.Mutex
token []byte
expires time.Time
}
type actorRef struct{ atespace, name string }
// FetchSecret returns the Biscuit for the actor the gateway attested. The URI
// must be ate-secret://<provider-name>/<instance-name>/biscuit; anything else
// is not served.
func (p *provider) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecretRequest) (*credproviderpb.FetchSecretResponse, error) {
if err := p.checkURI(req.GetUri()); err != nil {
slog.Warn("refused: not this provider's credential", slog.String("actor", req.GetActorSpiffeId()), slog.String("uri", req.GetUri()), slog.Any("err", err))
return nil, status.Error(codes.NotFound, err.Error())
}
actor, err := p.parseActor(req.GetActorSpiffeId())
if err != nil {
slog.Warn("refused: not an actor of this trust domain", slog.String("actor", req.GetActorSpiffeId()), slog.Any("err", err))
return nil, status.Error(codes.PermissionDenied, err.Error())
}
token, err := p.credential(ctx, req.GetActorSpiffeId(), actor)
if err != nil {
// mint logs a refusal by the mesh with its status; this is every other
// way the credential did not come (the node unreachable, the signer).
slog.Warn("no credential for the actor", slog.String("actor", req.GetActorSpiffeId()), slog.Any("err", err))
return nil, err
}
return &credproviderpb.FetchSecretResponse{OpaqueBytes: token}, nil
}
// cacheSize is the number of actors with a credential entry, read from the
// live map for the gauge on /metrics.
func (p *provider) cacheSize() int {
p.mu.Lock()
defer p.mu.Unlock()
return len(p.cache)
}
func (p *provider) uri() string {
return "ate-secret://" + p.name + "/" + p.instance + "/" + credentialTail
}
func (p *provider) checkURI(raw string) error {
u, err := url.Parse(raw)
if err != nil {
return fmt.Errorf("credential URI: %w", err)
}
if u.Scheme != "ate-secret" || u.Host != p.name || u.Path != "/"+p.instance+"/"+credentialTail || u.RawQuery != "" {
return fmt.Errorf("credential URI %q is not %s", raw, p.uri())
}
return nil
}
// parseActor accepts spiffe://<trust-domain>/actor/<atespace>/<name>, the ID
// Substrate's gateway derives from the actor's certificate.
func (p *provider) parseActor(id string) (actorRef, error) {
u, err := url.Parse(id)
if err != nil {
return actorRef{}, fmt.Errorf("actor SPIFFE ID: %w", err)
}
parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/")
if u.Scheme != "spiffe" || u.Host != p.trustDomain || len(parts) != 3 || parts[0] != "actor" || parts[1] == "" || parts[2] == "" {
return actorRef{}, fmt.Errorf("actor SPIFFE ID %q is not spiffe://%s/actor/<atespace>/<name>", id, p.trustDomain)
}
return actorRef{atespace: parts[1], name: parts[2]}, nil
}
// credential returns the cached Biscuit for subject, minting one when the
// cache has none or the entry has aged out. Concurrent requests for one actor
// share a single mint.
func (p *provider) credential(ctx context.Context, subject string, actor actorRef) ([]byte, error) {
p.mu.Lock()
entry, ok := p.cache[subject]
if !ok {
entry = &cached{}
p.cache[subject] = entry
}
p.mu.Unlock()
entry.mu.Lock()
defer entry.mu.Unlock()
if entry.token != nil && time.Now().Before(entry.expires) {
return entry.token, nil
}
token, lifetime, err := p.mint(ctx, subject, actor)
if err != nil {
return nil, err
}
reuse := min(lifetime-expirySkew, p.cacheTTL)
if reuse <= 0 {
return token, nil
}
entry.token, entry.expires = token, time.Now().Add(reuse)
return token, nil
}
// mint signs an assertion for subject and exchanges it at the node token
// endpoint. The control plane verifies the assertion against this provider's
// JWKS and resolves the subject's roles from the mesh policy.
func (p *provider) mint(ctx context.Context, subject string, actor actorRef) ([]byte, time.Duration, error) {
assertion, err := p.signer.assertion(subject, map[string]string{"atespace": actor.atespace, "actorName": actor.name})
if err != nil {
return nil, 0, status.Errorf(codes.Internal, "signing assertion: %v", err)
}
form := url.Values{
"grant_type": {grantTypeTokenExchange},
"subject_token": {assertion},
"subject_token_type": {tokenTypeJWT},
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.node+"/oauth/token", strings.NewReader(form.Encode()))
if err != nil {
return nil, 0, status.Errorf(codes.Internal, "token request: %v", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := p.http.Do(req)
if err != nil {
return nil, 0, status.Errorf(codes.Unavailable, "agentmesh-node token endpoint: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return nil, 0, status.Errorf(codes.Unavailable, "reading token response: %v", err)
}
var out struct {
AccessToken string `json:"access_token"`
ExpiresIn int64 `json:"expires_in"`
Error string `json:"error"`
Description string `json:"error_description"`
}
if err := json.Unmarshal(body, &out); err != nil {
return nil, 0, status.Errorf(codes.Unavailable, "token response is not JSON (HTTP %d)", resp.StatusCode)
}
if resp.StatusCode != http.StatusOK || out.AccessToken == "" {
slog.Warn("mesh refused the actor", slog.String("actor", subject), slog.Int("status", resp.StatusCode),
slog.String("error", out.Error), slog.String("description", out.Description))
return nil, 0, status.Errorf(codes.PermissionDenied, "mesh refused %s: %s %s", subject, out.Error, out.Description)
}
slog.Info("minted mesh credential", slog.String("actor", subject), slog.Int64("expires_in", out.ExpiresIn))
return []byte(out.AccessToken), time.Duration(out.ExpiresIn) * time.Second, nil
}
// signer holds the ES256 key the provider asserts actor identities with and
// publishes it the way an OIDC issuer does.
type signer struct {
issuer, audience string
kid string
key *ecdsa.PrivateKey
jwks, discovery []byte
}
// newSigner signs with the P-256 key at keyPath, or with one generated for
// this run when keyPath is empty. A persistent key is what lets the
// discovery document and the JWKS be published as static files somewhere
// the control plane can read (a bucket), with the issuer naming that place
// and jwksURI the JWKS there; an ephemeral key needs the issuer to be this
// process, since the keys change with every start.
func newSigner(issuer, audience, keyPath, jwksURI string) (*signer, error) {
var key *ecdsa.PrivateKey
if keyPath == "" {
var err error
if key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader); err != nil {
return nil, err
}
} else {
var err error
if key, err = loadP256Key(keyPath); err != nil {
return nil, err
}
}
if jwksURI == "" {
jwksURI = issuer + "/keys"
}
der, err := x509.MarshalPKIXPublicKey(&key.PublicKey)
if err != nil {
return nil, err
}
sum := sha256.Sum256(der)
s := &signer{issuer: issuer, audience: audience, kid: base64.RawURLEncoding.EncodeToString(sum[:]), key: key}
coord := func(b []byte) string {
padded := make([]byte, 32)
copy(padded[32-len(b):], b)
return base64.RawURLEncoding.EncodeToString(padded)
}
s.jwks, _ = json.Marshal(map[string]any{"keys": []map[string]string{{
"kty": "EC", "crv": "P-256", "alg": "ES256", "use": "sig", "kid": s.kid,
"x": coord(key.PublicKey.X.Bytes()), "y": coord(key.PublicKey.Y.Bytes()),
}}})
s.discovery, _ = json.Marshal(map[string]any{
"issuer": issuer,
"jwks_uri": jwksURI,
"response_types_supported": []string{"id_token"},
"subject_types_supported": []string{"public"},
"id_token_signing_alg_values_supported": []string{"ES256"},
"claims_supported": []string{"iss", "sub", "aud", "exp", "iat", "nbf", "jti", "ate.dev"},
})
return s, nil
}
// loadP256Key reads one PEM private key, PKCS#8 or SEC 1, and requires P-256.
func loadP256Key(path string) (*ecdsa.PrivateKey, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read signing key: %w", err)
}
block, _ := pem.Decode(data)
if block == nil {
return nil, fmt.Errorf("signing key %s: no PEM block", path)
}
var parsed any
switch block.Type {
case "PRIVATE KEY":
parsed, err = x509.ParsePKCS8PrivateKey(block.Bytes)
case "EC PRIVATE KEY":
parsed, err = x509.ParseECPrivateKey(block.Bytes)
default:
return nil, fmt.Errorf("signing key %s: unsupported PEM block %q", path, block.Type)
}
if err != nil {
return nil, fmt.Errorf("signing key %s: %w", path, err)
}
key, ok := parsed.(*ecdsa.PrivateKey)
if !ok || key.Curve != elliptic.P256() {
return nil, fmt.Errorf("signing key %s: want a P-256 key, got %T", path, parsed)
}
return key, nil
}
// writeDiscovery writes the two documents an OIDC relying party fetches,
// at the paths they are fetched from, so a bucket mirrors the issuer.
func (s *signer) writeDiscovery(dir string) error {
wk := filepath.Join(dir, ".well-known")
if err := os.MkdirAll(wk, 0o755); err != nil {
return err
}
if err := os.WriteFile(filepath.Join(wk, "openid-configuration"), s.discovery, 0o644); err != nil {
return err
}
return os.WriteFile(filepath.Join(dir, "keys"), s.jwks, 0o644)
}
func (s *signer) assertion(subject string, actor map[string]string) (string, error) {
now := time.Now()
jti := make([]byte, 16)
if _, err := rand.Read(jti); err != nil {
return "", err
}
token := jwt.NewWithClaims(jwt.SigningMethodES256, jwt.MapClaims{
"iss": s.issuer,
"sub": subject,
"aud": []string{s.audience},
"iat": now.Unix(),
"nbf": now.Add(-time.Minute).Unix(),
"exp": now.Add(assertionLifetime).Unix(),
"jti": hex.EncodeToString(jti),
"ate.dev": actor,
})
token.Header["kid"] = s.kid
return token.SignedString(s.key)
}
// handler serves the issuer documents, the probes, and /metrics with the
// one piece of state this process holds: the actors it has a credential
// cached for, read from the live map.
func (s *signer) handler(cacheSize func() int) http.Handler {
mux := http.NewServeMux()
serve := func(body []byte) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
_, _ = w.Write(body)
}
}
mux.Handle("GET /.well-known/openid-configuration", serve(s.discovery))
mux.Handle("GET /keys", serve(s.jwks))
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })
mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })
mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
fmt.Fprintf(w, "# HELP agentmesh_credential_provider_cached_actors Actors this provider holds a mesh credential for\n")
fmt.Fprintf(w, "# TYPE agentmesh_credential_provider_cached_actors gauge\n")
fmt.Fprintf(w, "agentmesh_credential_provider_cached_actors %d\n", cacheSize())
})
return mux
}
// serverCredentials builds the mutual TLS the gateway expects from a provider:
// the pod's servicedns certificate on the server side, and on the client side
// a certificate that chains to the pod-identity CA and carries the egress
// gateway's SPIFFE ID. Both files rotate, so they are read on every handshake.
func serverCredentials(bundlePath, caPath, expectedSAN string) (credentials.TransportCredentials, error) {
if _, err := loadBundle(bundlePath); err != nil {
return nil, err
}
if _, err := loadPool(caPath); err != nil {
return nil, err
}
cfg := &tls.Config{
MinVersion: tls.VersionTLS13,
GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {
pool, err := loadPool(caPath)
if err != nil {
return nil, err
}
return &tls.Config{
MinVersion: tls.VersionTLS13,
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: pool,
GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
return loadBundle(bundlePath)
},
VerifyConnection: func(state tls.ConnectionState) error {
if len(state.PeerCertificates) == 0 {
return errors.New("client certificate is required")
}
for _, u := range state.PeerCertificates[0].URIs {
if u.String() == expectedSAN {
return nil
}
}
return fmt.Errorf("client certificate URI SANs %v do not include %q", state.PeerCertificates[0].URIs, expectedSAN)
},
}, nil
},
}
return credentials.NewTLS(cfg), nil
}
// loadBundle reads a Kubernetes pod certificate credential bundle: a PEM
// private key followed by the certificate chain.
func loadBundle(path string) (*tls.Certificate, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("credential bundle: %w", err)
}
var cert tls.Certificate
for {
var block *pem.Block
block, data = pem.Decode(data)
if block == nil {
break
}
switch {
case block.Type == "CERTIFICATE":
cert.Certificate = append(cert.Certificate, block.Bytes)
case strings.HasSuffix(block.Type, "PRIVATE KEY") && cert.PrivateKey == nil:
key, err := parsePrivateKey(block)
if err != nil {
return nil, fmt.Errorf("credential bundle %s: %w", path, err)
}
cert.PrivateKey = key
}
}
if cert.PrivateKey == nil || len(cert.Certificate) == 0 {
return nil, fmt.Errorf("credential bundle %s has no key and certificate", path)
}
return &cert, nil
}
func parsePrivateKey(block *pem.Block) (any, error) {
if key, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil {
return key, nil
}
if key, err := x509.ParseECPrivateKey(block.Bytes); err == nil {
return key, nil
}
return x509.ParsePKCS1PrivateKey(block.Bytes)
}
func loadPool(path string) (*x509.CertPool, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("client CA bundle: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(data) {
return nil, fmt.Errorf("client CA bundle %s holds no certificates", path)
}
return pool, nil
}