Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane: resolve workload issuers in the background instead of failing to start
Antonio Ojeaf3bb5f5d3a32
diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.goindex 47d382dd..6f7fb2b0 100644--- a/internal/controlplane/metrics.go+++ b/internal/controlplane/metrics.go@@ -46,6 +46,17 @@ var (},[]string{"route"},)++ // workloadIssuerReady is 1 when the OpenID discovery document of a+ // --workload-issuer has been resolved and tokens from it are accepted,+ // 0 while the control plane is still trying and refuses them.+ workloadIssuerReady = promauto.NewGaugeVec(+ prometheus.GaugeOpts{+ Name: "agentmesh_control_plane_workload_issuer_ready",+ Help: "1 when the workload issuer's OpenID discovery has been resolved and its tokens are accepted, 0 while it is unreachable",+ },+ []string{"issuer"},+ ))// observeRoute counts and times requests to one registered mux pattern. Thediff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex 9b4537a4..b3ab5efa 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -30,6 +30,7 @@ import (mathrand "math/rand/v2""net""net/http"+ "sort""strconv""strings""sync"@@ -403,46 +404,130 @@ func meshSurface(h http.HandlerFunc) http.HandlerFunc {}}+// discoverProviders resolves the OpenID discovery document of every issuer.+//+// The login issuers (--issuer) are operator-fixed and the control plane is+// useless without them, so a failure there, after a bounded retry, is fatal:+// a typo must stop the process where it is seen. The workload issuers+// (--workload-issuer) come and go with the platforms that use them, often+// in another cluster or cloud, and one being unreachable must not take down+// enrollment, policy sync and token minting for everyone else. Those get one+// bounded attempt here and, on failure, a background resolver that keeps+// trying; until it succeeds a token from that issuer is refused with a line+// that names it, and the gauge agentmesh_control_plane_workload_issuer_ready+// says so.func (s *Server) discoverProviders() error {- s.providersMu.Lock()- defer s.providersMu.Unlock()-- var issuers []string+ var required, deferred []stringseen := make(map[string]bool)- addIssuer := func(iss string) {+ for _, iss := range strings.Split(s.config.OIDCIssuer, ",") {iss = strings.TrimSpace(iss)if iss != "" && !seen[iss] {seen[iss] = true- issuers = append(issuers, iss)+ required = append(required, iss)}}- for _, iss := range strings.Split(s.config.OIDCIssuer, ",") {- addIssuer(iss)+ addWorkload := func(iss string) {+ iss = strings.TrimSpace(iss)+ if iss != "" && !seen[iss] {+ seen[iss] = true+ deferred = append(deferred, iss)+ }}for iss := range s.workloadIssuers {- addIssuer(iss)+ addWorkload(iss)}for iss := range s.workloadEmailSuffixes {- addIssuer(iss)+ addWorkload(iss)}+ sort.Strings(deferred)- for _, iss := range issuers {- tr := http.DefaultTransport.(*http.Transport).Clone()- tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: s.config.InsecureSkipTLSVerify}- client := &http.Client{- Timeout: 30 * time.Second,- Transport: tr,- }- providerCtx := oidc.ClientContext(s.ctx, client)- provider, err := discoverProviderWithRetry(providerCtx, iss, oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay)+ for _, iss := range required {+ provider, err := s.discoverProvider(iss)if err != nil {return fmt.Errorf("failed to create provider for %s: %w", iss, err)}- s.providers[iss] = provider+ s.setProvider(iss, provider)+ }+ for _, iss := range deferred {+ workloadIssuerReady.WithLabelValues(iss).Set(0)+ provider, err := s.discoverProvider(iss)+ if err != nil {+ logger.Warnw("Workload issuer not resolved; tokens from it are refused until it answers", "issuer", iss, "error", err)+ s.wg.Add(1)+ go s.resolveWorkloadIssuer(iss)+ continue+ }+ s.setProvider(iss, provider)}return nil}+// discoverProvider runs one bounded discovery of iss with the server's HTTP+// settings.+func (s *Server) discoverProvider(iss string) (*oidc.Provider, error) {+ tr := http.DefaultTransport.(*http.Transport).Clone()+ tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: s.config.InsecureSkipTLSVerify}+ client := &http.Client{+ Timeout: 30 * time.Second,+ Transport: tr,+ }+ providerCtx := oidc.ClientContext(s.ctx, client)+ return discoverProviderWithRetry(providerCtx, iss, oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay)+}++func (s *Server) setProvider(iss string, provider *oidc.Provider) {+ s.providersMu.Lock()+ defer s.providersMu.Unlock()+ s.providers[iss] = provider+ if s.workloadIssuers[iss] || len(s.workloadEmailSuffixes[iss]) > 0 {+ workloadIssuerReady.WithLabelValues(iss).Set(1)+ }+}++// Backoff for the background resolver of a workload issuer that did not+// answer at start-up. Package variables so a test can shorten them.+var (+ workloadIssuerRetryBase = 5 * time.Second+ workloadIssuerRetryMax = 5 * time.Minute+)++// resolveWorkloadIssuer keeps trying the discovery of iss until it succeeds+// or the server stops, with exponential backoff capped at+// workloadIssuerRetryMax. Each failure is logged so the gap is readable from+// the logs as well as from the gauge.+func (s *Server) resolveWorkloadIssuer(iss string) {+ defer s.wg.Done()+ delay := workloadIssuerRetryBase+ for attempt := 1; ; attempt++ {+ select {+ case <-s.ctx.Done():+ return+ case <-time.After(delay):+ }+ provider, err := s.discoverProvider(iss)+ if err == nil {+ s.setProvider(iss, provider)+ logger.Infow("Workload issuer resolved; tokens from it are accepted", "issuer", iss, "attempts", attempt)+ return+ }+ logger.Warnw("Workload issuer still not resolved", "issuer", iss, "attempt", attempt, "next_retry", delay.String(), "error", err)+ if delay *= 2; delay > workloadIssuerRetryMax {+ delay = workloadIssuerRetryMax+ }+ }+}++// jwtRefusalMessage turns a VerifyJWT error into the message the caller+// receives, distinguishing a configured workload issuer whose discovery has+// not succeeded yet from an issuer that was never configured.+func (s *Server) jwtRefusalMessage(err error) string {+ var unknown *identity.UnknownIssuerError+ if errors.As(err, &unknown) && (s.workloadIssuers[unknown.Issuer] || len(s.workloadEmailSuffixes[unknown.Issuer]) > 0) {+ return fmt.Sprintf("JWT validation failed: workload issuer %s is configured but its discovery document has not been resolved yet", unknown.Issuer)+ }+ return "JWT validation failed: " + err.Error()+}+// isWorkloadClaims reports whether verified JWT claims belong to a machine or// workload identity rather than a human operator:// 1. The token's iss is a dedicated workload issuer (--workload-issuer=<iss>),@@ -484,7 +569,8 @@ func (s *Server) sessionTTLForClaims(claims jwt.MapClaims) time.Duration {// Defaults for discoverProviderWithRetry; kept small enough that a real outage still// surfaces quickly (worst case ~15s) while riding out a transient hiccup during rollouts.-const (+// Package variables so a test can shorten them.+var (oidcDiscoveryMaxAttempts = 5oidcDiscoveryBaseDelay = 1 * time.SecondoidcDiscoveryMaxDelay = 8 * time.Second@@ -749,7 +835,7 @@ func (s *Server) HandleRegister(w http.ResponseWriter, r *http.Request) {claims, token, err := identity.VerifyJWT(ctx, req.Jwt, s.config.AllowedAudiences, s.getProviders())if err != nil {logger.Errorw("JWT verification failed", "peer_id", req.PeerId, "error", err)- http.Error(w, "JWT validation failed: "+err.Error(), http.StatusUnauthorized)+ http.Error(w, s.jwtRefusalMessage(err), http.StatusUnauthorized)return}// Bans and ownership are keyed on issuer|subject; a token without a@@ -1135,7 +1221,7 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {cancel()if err != nil {logger.Warnw("Refresh JWT verification failed", "peer_id", canonical, "error", err)- http.Error(w, "JWT validation failed: "+err.Error(), http.StatusUnauthorized)+ http.Error(w, s.jwtRefusalMessage(err), http.StatusUnauthorized)return}if verifiedEmail(freshClaims) == "" {diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.goindex abc830fd..28407831 100644--- a/internal/controlplane/server_test.go+++ b/internal/controlplane/server_test.go@@ -46,6 +46,8 @@ import ("github.com/google/agentmesh/internal/storage""github.com/libp2p/go-libp2p/core/crypto""github.com/libp2p/go-libp2p/core/peer"+ "github.com/prometheus/client_golang/prometheus"+ dto "github.com/prometheus/client_model/go""google.golang.org/protobuf/encoding/protojson""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/durationpb"@@ -4786,3 +4788,199 @@ func TestOptionsRefuseAnUnusableDirectory(t *testing.T) {})}}++// startTogglableMockOIDC is startCustomMockOIDC with a switch: while down, the+// discovery document answers 503, the way an identity provider that has not+// been deployed yet, or is mid-rollout, does.+func startTogglableMockOIDC(t *testing.T) (issuer string, mint func(map[string]interface{}) string, setUp func(bool)) {+ t.Helper()+ privKey, err := rsa.GenerateKey(rand.Reader, 2048)+ if err != nil {+ t.Fatalf("failed to generate rsa key: %v", err)+ }+ var up atomic.Bool+ mux := http.NewServeMux()+ srv := httptest.NewServer(mux)+ t.Cleanup(srv.Close)+ issuer = srv.URL++ mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {+ if !up.Load() {+ http.Error(w, "not deployed yet", http.StatusServiceUnavailable)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "issuer": issuer,+ "jwks_uri": issuer + "/keys",+ })+ })+ mux.HandleFunc("/keys", func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "keys": []map[string]interface{}{{+ "kty": "RSA", "alg": "RS256", "use": "sig", "kid": "mock-key",+ "n": base64.RawURLEncoding.EncodeToString(privKey.N.Bytes()),+ "e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(privKey.E)).Bytes()),+ }},+ })+ })+ mint = func(custom map[string]interface{}) string {+ claims := jwt.MapClaims{"iss": issuer, "aud": "agentmesh-audience", "exp": time.Now().Add(time.Hour).Unix()}+ for k, v := range custom {+ claims[k] = v+ }+ tok := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)+ tok.Header["kid"] = "mock-key"+ signed, err := tok.SignedString(privKey)+ if err != nil {+ t.Fatalf("sign token: %v", err)+ }+ return signed+ }+ return issuer, mint, up.Store+}++// gaugeValue reads a gauge the way a scraper would.+func gaugeValue(t *testing.T, g prometheus.Gauge) float64 {+ t.Helper()+ var m dto.Metric+ if err := g.Write(&m); err != nil {+ t.Fatalf("read gauge: %v", err)+ }+ return m.GetGauge().GetValue()+}++// shortenDiscoveryBackoff makes start-up discovery and the background+// resolver fast enough for a test, restoring the defaults afterwards.+func shortenDiscoveryBackoff(t *testing.T) {+ t.Helper()+ attempts, base, maxDelay := oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay+ retryBase, retryMax := workloadIssuerRetryBase, workloadIssuerRetryMax+ oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay = 2, time.Millisecond, 5*time.Millisecond+ workloadIssuerRetryBase, workloadIssuerRetryMax = 20*time.Millisecond, 100*time.Millisecond+ t.Cleanup(func() {+ oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay = attempts, base, maxDelay+ workloadIssuerRetryBase, workloadIssuerRetryMax = retryBase, retryMax+ })+}++// TestLoginIssuerDiscoveryFailureIsFatal keeps the operator-facing issuer+// strict: a --issuer that does not answer stops the control plane, so that a+// typo is seen where it is made.+func TestLoginIssuerDiscoveryFailureIsFatal(t *testing.T) {+ shortenDiscoveryBackoff(t)+ issuer, _, _ := startTogglableMockOIDC(t) // stays down++ store, err := storage.NewSQLStore("sqlite", ":memory:")+ if err != nil {+ t.Fatalf("NewSQLStore: %v", err)+ }+ defer func() { _ = store.Close() }()+ srv, err := NewServer(Options{+ ListenAddr: "127.0.0.1:0",+ OIDCIssuer: issuer,+ AllowedAudiences: []string{"agentmesh-audience"},+ }, store)+ if err != nil {+ t.Fatalf("NewServer: %v", err)+ }+ err = srv.Start()+ if err == nil {+ _ = srv.Close()+ t.Fatal("Start succeeded with an unreachable --issuer, want an error")+ }+ if !strings.Contains(err.Error(), "failed to create provider for "+issuer) {+ t.Fatalf("Start error = %q, want it to name the issuer %s", err, issuer)+ }+}++// TestWorkloadIssuerResolvedInBackground is the Cloud Run case: the hub is+// deployed before the platform whose identities it will accept exists. The+// control plane must start, refuse tokens from that issuer with a message+// that says why, report it on the gauge, and accept them once the issuer+// answers, without a restart.+func TestWorkloadIssuerResolvedInBackground(t *testing.T) {+ shortenDiscoveryBackoff(t)+ loginIss, mintLogin := startCustomMockOIDC(t)+ workloadIss, mintWorkload, setWorkloadUp := startTogglableMockOIDC(t)++ srv, store, baseURL := setupTestServer(t, loginIss, func(o *Options) {+ o.WorkloadIssuer = workloadIss+ })+ defer func() { _ = srv.Close() }()++ if got := gaugeValue(t, workloadIssuerReady.WithLabelValues(workloadIss)); got != 0 {+ t.Fatalf("workload_issuer_ready{%s} = %v at start with the issuer down, want 0", workloadIss, got)+ }+ if _, ok := srv.getProviders()[workloadIss]; ok {+ t.Fatalf("provider for %s present although its discovery never answered", workloadIss)+ }++ ctx := context.Background()+ if err := store.SavePolicyDocument(ctx,+ []*api.PolicyRole{{Name: api.RoleNode, AllowedServices: []string{"*"}, AllowedTargets: []string{"*"}}},+ []*api.PolicyBinding{{Role: api.RoleNode, Members: []string{api.SystemAuthenticated}}}, nil); err != nil {+ t.Fatalf("SavePolicyDocument: %v", err)+ }+ nodePriv, nodePeerID, nodeBiscuit, _ := enrollTestNodeForSTS(t, baseURL, mintLogin)++ exchange := func() (int, string) {+ t.Helper()+ ts := time.Now().UnixMilli()+ sig, err := nodePriv.Sign(api.TokenExchangeChallenge(nodePeerID.String(), ts))+ if err != nil {+ t.Fatalf("sign challenge: %v", err)+ }+ body, err := proto.Marshal(&api.TokenExchangeRequest{+ SubjectToken: mintWorkload(map[string]interface{}{"sub": "spiffe://platform.example/workload/a"}),+ ChallengeUnixMs: ts,+ ChallengeSignature: sig,+ })+ if err != nil {+ t.Fatalf("Marshal TokenExchangeRequest: %v", err)+ }+ req, err := http.NewRequest(http.MethodPost, baseURL+"/token/exchange", bytes.NewReader(body))+ if err != nil {+ t.Fatalf("NewRequest: %v", err)+ }+ req.Header.Set("Content-Type", "application/x-protobuf")+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(nodeBiscuit))+ resp, err := http.DefaultClient.Do(req)+ if err != nil {+ t.Fatalf("POST /token/exchange: %v", err)+ }+ defer func() { _ = resp.Body.Close() }()+ respBody, err := io.ReadAll(resp.Body)+ if err != nil {+ t.Fatalf("read /token/exchange body: %v", err)+ }+ return resp.StatusCode, string(respBody)+ }++ code, body := exchange()+ if code != http.StatusUnauthorized {+ t.Fatalf("exchange with the issuer unresolved returned %d %q, want 401", code, body)+ }+ if !strings.Contains(body, "workload issuer "+workloadIss+" is configured but its discovery document has not been resolved yet") {+ t.Fatalf("exchange refusal = %q, want it to name the unresolved issuer", body)+ }++ setWorkloadUp(true)+ deadline := time.Now().Add(5 * time.Second)+ for {+ if _, ok := srv.getProviders()[workloadIss]; ok {+ break+ }+ if time.Now().After(deadline) {+ t.Fatalf("provider for %s not resolved within 5s of the issuer coming up", workloadIss)+ }+ time.Sleep(10 * time.Millisecond)+ }+ if got := gaugeValue(t, workloadIssuerReady.WithLabelValues(workloadIss)); got != 1 {+ t.Fatalf("workload_issuer_ready{%s} = %v after resolution, want 1", workloadIss, got)+ }+ if code, body := exchange(); code != http.StatusOK {+ t.Fatalf("exchange after the issuer resolved returned %d %q, want 200", code, body)+ }+}diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.goindex 37ccaeb3..81262125 100644--- a/internal/controlplane/sts.go+++ b/internal/controlplane/sts.go@@ -859,7 +859,7 @@ func (s *Server) mintDelegatedBiscuitFromJWT(claims, token, err := identity.VerifyJWT(verifyCtx, subjectJWT, s.config.AllowedAudiences, s.getProviders())if err != nil {- return nil, time.Time{}, "", nil, http.StatusUnauthorized, fmt.Errorf("JWT validation failed: %w", err)+ return nil, time.Time{}, "", nil, http.StatusUnauthorized, errors.New(s.jwtRefusalMessage(err))}if !allowWorkload && s.isWorkloadClaims(claims) {return nil, time.Time{}, "", nil, http.StatusForbidden, errWorkloadIdentitydiff --git a/internal/identity/oidc.go b/internal/identity/oidc.goindex 7d38ab1e..9831f508 100644--- a/internal/identity/oidc.go+++ b/internal/identity/oidc.go@@ -23,6 +23,17 @@ import ("github.com/golang-jwt/jwt/v5")+// UnknownIssuerError is returned by VerifyJWT when the token names an issuer+// that has no resolved provider: either one that was never configured, or one+// whose OpenID discovery has not succeeded yet. The caller tells the two apart.+type UnknownIssuerError struct {+ Issuer string+}++func (e *UnknownIssuerError) Error() string {+ return fmt.Sprintf("unknown issuer: %s", e.Issuer)+}+// VerifyJWT parses and cryptographically validates a JWT token against a list of allowed audiences// and resolved OIDC providers.func VerifyJWT(ctx context.Context, jwtStr string, allowedAudiences []string, providers map[string]*oidc.Provider) (jwt.MapClaims, *oidc.IDToken, error) {@@ -37,7 +48,7 @@ func VerifyJWT(ctx context.Context, jwtStr string, allowedAudiences []string, prprovider, ok := providers[iss]if !ok {- return nil, nil, fmt.Errorf("unknown issuer: %s", iss)+ return nil, nil, &UnknownIssuerError{Issuer: iss}}// SkipClientIDCheck because the verifier only knows how to match a singlediff --git a/site/content/docs/reference/agentmesh-one.md b/site/content/docs/reference/agentmesh-one.mdindex a58e9464..30387d15 100644--- a/site/content/docs/reference/agentmesh-one.md+++ b/site/content/docs/reference/agentmesh-one.md@@ -43,7 +43,7 @@ downloaded tunnel connector.| `--admin-token-path` | | File containing the admin token. Can also be set with `AGENTMESH_ADMIN_TOKEN`. Generated and saved to the data directory when neither is set. || `--policy-file` | | Protojson `PolicyConfig` that seeds the mesh policy on first boot. Ignored once the database has a policy. Without it, first boot seeds an open development policy and logs a warning. || `--issuer` | | External OIDC issuer(s), comma-separated. Optional. Without an issuer, enrollment works by token only. |-| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`). Automatically added to `--issuer` and refused at `/user/*` and `/oauth/authorize`. |+| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`). Automatically added to `--issuer` and refused at `/user/*` and `/oauth/authorize`. One that does not answer at start-up is resolved in the background and its tokens refused until then (`agentmesh_control_plane_workload_issuer_ready{issuer}`); a `--issuer` that does not answer is fatal. || `--oidc-client-id` | first audience | Client ID advertised on `/info`. || `--allowed-audiences` | `agentmesh-audience` | Accepted OIDC audiences. || `--enroll-qr` | when stdout is a terminal | Print a device-enrollment QR code at startup. Only for `https` URLs. |diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.mdindex c35086d3..faef6fa6 100644--- a/site/content/docs/reference/control-plane.md+++ b/site/content/docs/reference/control-plane.md@@ -20,7 +20,7 @@ agentmesh-control-plane admin unban --peer <id> lift a ban| Flag | Default | Meaning ||---|---|---|| `--issuer` | | OIDC issuer URL(s), comma-separated. At least one of `--issuer` or `--workload-issuer` is required. The first issuer is advertised to enrolling nodes on `/info`. |-| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`, such as `https://accounts.google.com=.gserviceaccount.com`). Automatically added to `--issuer`. Workload tokens can enroll, refresh, and exchange credentials (`/register`, `/refresh`, `/token/exchange`), and are refused at `/user/*` and `/oauth/authorize`. |+| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`, such as `https://accounts.google.com=.gserviceaccount.com`). Automatically added to `--issuer`. Workload tokens can enroll, refresh, and exchange credentials (`/register`, `/refresh`, `/token/exchange`), and are refused at `/user/*` and `/oauth/authorize`. A workload issuer that does not answer its discovery document at start-up does not stop the control plane: it keeps retrying in the background (backoff up to 5 minutes), refuses tokens from that issuer with `401` and a message naming it until it does, and reports it as `agentmesh_control_plane_workload_issuer_ready{issuer}` = 0. An `--issuer` that does not answer is still fatal. || `--allowed-audiences` | `agentmesh-audience` | Audiences accepted in OIDC tokens, comma-separated. || `--oidc-client-id` | first audience | OAuth client ID advertised on `/info`, for providers where it differs from the audience. || `--insecure-skip-tls-verify` | `false` | Skip TLS verification when fetching issuer metadata and keys. For a cluster issuer served with the cluster CA, or a local development issuer. |