Skip to content

Agent Mesh

controlplane: resolve workload issuers in the background instead of failing to start

Antonio Ojeaf3bb5f5d3a32
diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.go
index 47d382dd..6f7fb2b0 100644
--- a/internal/controlplane/metrics.go
+++ b/internal/controlplane/metrics.go
@@ -46,6 +46,17 @@ var (
},
[]string{"route"},
)
+
+ // workloadIssuerReady is 1 when the OpenID discovery document of a
+ // --workload-issuer has been resolved and tokens from it are accepted,
+ // 0 while the control plane is still trying and refuses them.
+ workloadIssuerReady = promauto.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Name: "agentmesh_control_plane_workload_issuer_ready",
+ Help: "1 when the workload issuer's OpenID discovery has been resolved and its tokens are accepted, 0 while it is unreachable",
+ },
+ []string{"issuer"},
+ )
)
// observeRoute counts and times requests to one registered mux pattern. The
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 9b4537a4..b3ab5efa 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -30,6 +30,7 @@ import (
mathrand "math/rand/v2"
"net"
"net/http"
+ "sort"
"strconv"
"strings"
"sync"
@@ -403,46 +404,130 @@ func meshSurface(h http.HandlerFunc) http.HandlerFunc {
}
}
+// discoverProviders resolves the OpenID discovery document of every issuer.
+//
+// The login issuers (--issuer) are operator-fixed and the control plane is
+// useless without them, so a failure there, after a bounded retry, is fatal:
+// a typo must stop the process where it is seen. The workload issuers
+// (--workload-issuer) come and go with the platforms that use them, often
+// in another cluster or cloud, and one being unreachable must not take down
+// enrollment, policy sync and token minting for everyone else. Those get one
+// bounded attempt here and, on failure, a background resolver that keeps
+// trying; until it succeeds a token from that issuer is refused with a line
+// that names it, and the gauge agentmesh_control_plane_workload_issuer_ready
+// says so.
func (s *Server) discoverProviders() error {
- s.providersMu.Lock()
- defer s.providersMu.Unlock()
-
- var issuers []string
+ var required, deferred []string
seen := make(map[string]bool)
- addIssuer := func(iss string) {
+ for _, iss := range strings.Split(s.config.OIDCIssuer, ",") {
iss = strings.TrimSpace(iss)
if iss != "" && !seen[iss] {
seen[iss] = true
- issuers = append(issuers, iss)
+ required = append(required, iss)
}
}
- for _, iss := range strings.Split(s.config.OIDCIssuer, ",") {
- addIssuer(iss)
+ addWorkload := func(iss string) {
+ iss = strings.TrimSpace(iss)
+ if iss != "" && !seen[iss] {
+ seen[iss] = true
+ deferred = append(deferred, iss)
+ }
}
for iss := range s.workloadIssuers {
- addIssuer(iss)
+ addWorkload(iss)
}
for iss := range s.workloadEmailSuffixes {
- addIssuer(iss)
+ addWorkload(iss)
}
+ sort.Strings(deferred)
- for _, iss := range issuers {
- tr := http.DefaultTransport.(*http.Transport).Clone()
- tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: s.config.InsecureSkipTLSVerify}
- client := &http.Client{
- Timeout: 30 * time.Second,
- Transport: tr,
- }
- providerCtx := oidc.ClientContext(s.ctx, client)
- provider, err := discoverProviderWithRetry(providerCtx, iss, oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay)
+ for _, iss := range required {
+ provider, err := s.discoverProvider(iss)
if err != nil {
return fmt.Errorf("failed to create provider for %s: %w", iss, err)
}
- s.providers[iss] = provider
+ s.setProvider(iss, provider)
+ }
+ for _, iss := range deferred {
+ workloadIssuerReady.WithLabelValues(iss).Set(0)
+ provider, err := s.discoverProvider(iss)
+ if err != nil {
+ logger.Warnw("Workload issuer not resolved; tokens from it are refused until it answers", "issuer", iss, "error", err)
+ s.wg.Add(1)
+ go s.resolveWorkloadIssuer(iss)
+ continue
+ }
+ s.setProvider(iss, provider)
}
return nil
}
+// discoverProvider runs one bounded discovery of iss with the server's HTTP
+// settings.
+func (s *Server) discoverProvider(iss string) (*oidc.Provider, error) {
+ tr := http.DefaultTransport.(*http.Transport).Clone()
+ tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: s.config.InsecureSkipTLSVerify}
+ client := &http.Client{
+ Timeout: 30 * time.Second,
+ Transport: tr,
+ }
+ providerCtx := oidc.ClientContext(s.ctx, client)
+ return discoverProviderWithRetry(providerCtx, iss, oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay)
+}
+
+func (s *Server) setProvider(iss string, provider *oidc.Provider) {
+ s.providersMu.Lock()
+ defer s.providersMu.Unlock()
+ s.providers[iss] = provider
+ if s.workloadIssuers[iss] || len(s.workloadEmailSuffixes[iss]) > 0 {
+ workloadIssuerReady.WithLabelValues(iss).Set(1)
+ }
+}
+
+// Backoff for the background resolver of a workload issuer that did not
+// answer at start-up. Package variables so a test can shorten them.
+var (
+ workloadIssuerRetryBase = 5 * time.Second
+ workloadIssuerRetryMax = 5 * time.Minute
+)
+
+// resolveWorkloadIssuer keeps trying the discovery of iss until it succeeds
+// or the server stops, with exponential backoff capped at
+// workloadIssuerRetryMax. Each failure is logged so the gap is readable from
+// the logs as well as from the gauge.
+func (s *Server) resolveWorkloadIssuer(iss string) {
+ defer s.wg.Done()
+ delay := workloadIssuerRetryBase
+ for attempt := 1; ; attempt++ {
+ select {
+ case <-s.ctx.Done():
+ return
+ case <-time.After(delay):
+ }
+ provider, err := s.discoverProvider(iss)
+ if err == nil {
+ s.setProvider(iss, provider)
+ logger.Infow("Workload issuer resolved; tokens from it are accepted", "issuer", iss, "attempts", attempt)
+ return
+ }
+ logger.Warnw("Workload issuer still not resolved", "issuer", iss, "attempt", attempt, "next_retry", delay.String(), "error", err)
+ if delay *= 2; delay > workloadIssuerRetryMax {
+ delay = workloadIssuerRetryMax
+ }
+ }
+}
+
+// jwtRefusalMessage turns a VerifyJWT error into the message the caller
+// receives, distinguishing a configured workload issuer whose discovery has
+// not succeeded yet from an issuer that was never configured.
+func (s *Server) jwtRefusalMessage(err error) string {
+ var unknown *identity.UnknownIssuerError
+ if errors.As(err, &unknown) && (s.workloadIssuers[unknown.Issuer] || len(s.workloadEmailSuffixes[unknown.Issuer]) > 0) {
+ return fmt.Sprintf("JWT validation failed: workload issuer %s is configured but its discovery document has not been resolved yet", unknown.Issuer)
+ }
+ return "JWT validation failed: " + err.Error()
+}
+
// isWorkloadClaims reports whether verified JWT claims belong to a machine or
// workload identity rather than a human operator:
// 1. The token's iss is a dedicated workload issuer (--workload-issuer=<iss>),
@@ -484,7 +569,8 @@ func (s *Server) sessionTTLForClaims(claims jwt.MapClaims) time.Duration {
// Defaults for discoverProviderWithRetry; kept small enough that a real outage still
// surfaces quickly (worst case ~15s) while riding out a transient hiccup during rollouts.
-const (
+// Package variables so a test can shorten them.
+var (
oidcDiscoveryMaxAttempts = 5
oidcDiscoveryBaseDelay = 1 * time.Second
oidcDiscoveryMaxDelay = 8 * time.Second
@@ -749,7 +835,7 @@ func (s *Server) HandleRegister(w http.ResponseWriter, r *http.Request) {
claims, token, err := identity.VerifyJWT(ctx, req.Jwt, s.config.AllowedAudiences, s.getProviders())
if err != nil {
logger.Errorw("JWT verification failed", "peer_id", req.PeerId, "error", err)
- http.Error(w, "JWT validation failed: "+err.Error(), http.StatusUnauthorized)
+ http.Error(w, s.jwtRefusalMessage(err), http.StatusUnauthorized)
return
}
// Bans and ownership are keyed on issuer|subject; a token without a
@@ -1135,7 +1221,7 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {
cancel()
if err != nil {
logger.Warnw("Refresh JWT verification failed", "peer_id", canonical, "error", err)
- http.Error(w, "JWT validation failed: "+err.Error(), http.StatusUnauthorized)
+ http.Error(w, s.jwtRefusalMessage(err), http.StatusUnauthorized)
return
}
if verifiedEmail(freshClaims) == "" {
diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.go
index abc830fd..28407831 100644
--- a/internal/controlplane/server_test.go
+++ b/internal/controlplane/server_test.go
@@ -46,6 +46,8 @@ import (
"github.com/google/agentmesh/internal/storage"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/durationpb"
@@ -4786,3 +4788,199 @@ func TestOptionsRefuseAnUnusableDirectory(t *testing.T) {
})
}
}
+
+// startTogglableMockOIDC is startCustomMockOIDC with a switch: while down, the
+// discovery document answers 503, the way an identity provider that has not
+// been deployed yet, or is mid-rollout, does.
+func startTogglableMockOIDC(t *testing.T) (issuer string, mint func(map[string]interface{}) string, setUp func(bool)) {
+ t.Helper()
+ privKey, err := rsa.GenerateKey(rand.Reader, 2048)
+ if err != nil {
+ t.Fatalf("failed to generate rsa key: %v", err)
+ }
+ var up atomic.Bool
+ mux := http.NewServeMux()
+ srv := httptest.NewServer(mux)
+ t.Cleanup(srv.Close)
+ issuer = srv.URL
+
+ mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
+ if !up.Load() {
+ http.Error(w, "not deployed yet", http.StatusServiceUnavailable)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "issuer": issuer,
+ "jwks_uri": issuer + "/keys",
+ })
+ })
+ mux.HandleFunc("/keys", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "keys": []map[string]interface{}{{
+ "kty": "RSA", "alg": "RS256", "use": "sig", "kid": "mock-key",
+ "n": base64.RawURLEncoding.EncodeToString(privKey.N.Bytes()),
+ "e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(privKey.E)).Bytes()),
+ }},
+ })
+ })
+ mint = func(custom map[string]interface{}) string {
+ claims := jwt.MapClaims{"iss": issuer, "aud": "agentmesh-audience", "exp": time.Now().Add(time.Hour).Unix()}
+ for k, v := range custom {
+ claims[k] = v
+ }
+ tok := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
+ tok.Header["kid"] = "mock-key"
+ signed, err := tok.SignedString(privKey)
+ if err != nil {
+ t.Fatalf("sign token: %v", err)
+ }
+ return signed
+ }
+ return issuer, mint, up.Store
+}
+
+// gaugeValue reads a gauge the way a scraper would.
+func gaugeValue(t *testing.T, g prometheus.Gauge) float64 {
+ t.Helper()
+ var m dto.Metric
+ if err := g.Write(&m); err != nil {
+ t.Fatalf("read gauge: %v", err)
+ }
+ return m.GetGauge().GetValue()
+}
+
+// shortenDiscoveryBackoff makes start-up discovery and the background
+// resolver fast enough for a test, restoring the defaults afterwards.
+func shortenDiscoveryBackoff(t *testing.T) {
+ t.Helper()
+ attempts, base, maxDelay := oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay
+ retryBase, retryMax := workloadIssuerRetryBase, workloadIssuerRetryMax
+ oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay = 2, time.Millisecond, 5*time.Millisecond
+ workloadIssuerRetryBase, workloadIssuerRetryMax = 20*time.Millisecond, 100*time.Millisecond
+ t.Cleanup(func() {
+ oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay = attempts, base, maxDelay
+ workloadIssuerRetryBase, workloadIssuerRetryMax = retryBase, retryMax
+ })
+}
+
+// TestLoginIssuerDiscoveryFailureIsFatal keeps the operator-facing issuer
+// strict: a --issuer that does not answer stops the control plane, so that a
+// typo is seen where it is made.
+func TestLoginIssuerDiscoveryFailureIsFatal(t *testing.T) {
+ shortenDiscoveryBackoff(t)
+ issuer, _, _ := startTogglableMockOIDC(t) // stays down
+
+ store, err := storage.NewSQLStore("sqlite", ":memory:")
+ if err != nil {
+ t.Fatalf("NewSQLStore: %v", err)
+ }
+ defer func() { _ = store.Close() }()
+ srv, err := NewServer(Options{
+ ListenAddr: "127.0.0.1:0",
+ OIDCIssuer: issuer,
+ AllowedAudiences: []string{"agentmesh-audience"},
+ }, store)
+ if err != nil {
+ t.Fatalf("NewServer: %v", err)
+ }
+ err = srv.Start()
+ if err == nil {
+ _ = srv.Close()
+ t.Fatal("Start succeeded with an unreachable --issuer, want an error")
+ }
+ if !strings.Contains(err.Error(), "failed to create provider for "+issuer) {
+ t.Fatalf("Start error = %q, want it to name the issuer %s", err, issuer)
+ }
+}
+
+// TestWorkloadIssuerResolvedInBackground is the Cloud Run case: the hub is
+// deployed before the platform whose identities it will accept exists. The
+// control plane must start, refuse tokens from that issuer with a message
+// that says why, report it on the gauge, and accept them once the issuer
+// answers, without a restart.
+func TestWorkloadIssuerResolvedInBackground(t *testing.T) {
+ shortenDiscoveryBackoff(t)
+ loginIss, mintLogin := startCustomMockOIDC(t)
+ workloadIss, mintWorkload, setWorkloadUp := startTogglableMockOIDC(t)
+
+ srv, store, baseURL := setupTestServer(t, loginIss, func(o *Options) {
+ o.WorkloadIssuer = workloadIss
+ })
+ defer func() { _ = srv.Close() }()
+
+ if got := gaugeValue(t, workloadIssuerReady.WithLabelValues(workloadIss)); got != 0 {
+ t.Fatalf("workload_issuer_ready{%s} = %v at start with the issuer down, want 0", workloadIss, got)
+ }
+ if _, ok := srv.getProviders()[workloadIss]; ok {
+ t.Fatalf("provider for %s present although its discovery never answered", workloadIss)
+ }
+
+ ctx := context.Background()
+ if err := store.SavePolicyDocument(ctx,
+ []*api.PolicyRole{{Name: api.RoleNode, AllowedServices: []string{"*"}, AllowedTargets: []string{"*"}}},
+ []*api.PolicyBinding{{Role: api.RoleNode, Members: []string{api.SystemAuthenticated}}}, nil); err != nil {
+ t.Fatalf("SavePolicyDocument: %v", err)
+ }
+ nodePriv, nodePeerID, nodeBiscuit, _ := enrollTestNodeForSTS(t, baseURL, mintLogin)
+
+ exchange := func() (int, string) {
+ t.Helper()
+ ts := time.Now().UnixMilli()
+ sig, err := nodePriv.Sign(api.TokenExchangeChallenge(nodePeerID.String(), ts))
+ if err != nil {
+ t.Fatalf("sign challenge: %v", err)
+ }
+ body, err := proto.Marshal(&api.TokenExchangeRequest{
+ SubjectToken: mintWorkload(map[string]interface{}{"sub": "spiffe://platform.example/workload/a"}),
+ ChallengeUnixMs: ts,
+ ChallengeSignature: sig,
+ })
+ if err != nil {
+ t.Fatalf("Marshal TokenExchangeRequest: %v", err)
+ }
+ req, err := http.NewRequest(http.MethodPost, baseURL+"/token/exchange", bytes.NewReader(body))
+ if err != nil {
+ t.Fatalf("NewRequest: %v", err)
+ }
+ req.Header.Set("Content-Type", "application/x-protobuf")
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(nodeBiscuit))
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("POST /token/exchange: %v", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ respBody, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("read /token/exchange body: %v", err)
+ }
+ return resp.StatusCode, string(respBody)
+ }
+
+ code, body := exchange()
+ if code != http.StatusUnauthorized {
+ t.Fatalf("exchange with the issuer unresolved returned %d %q, want 401", code, body)
+ }
+ if !strings.Contains(body, "workload issuer "+workloadIss+" is configured but its discovery document has not been resolved yet") {
+ t.Fatalf("exchange refusal = %q, want it to name the unresolved issuer", body)
+ }
+
+ setWorkloadUp(true)
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ if _, ok := srv.getProviders()[workloadIss]; ok {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("provider for %s not resolved within 5s of the issuer coming up", workloadIss)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ if got := gaugeValue(t, workloadIssuerReady.WithLabelValues(workloadIss)); got != 1 {
+ t.Fatalf("workload_issuer_ready{%s} = %v after resolution, want 1", workloadIss, got)
+ }
+ if code, body := exchange(); code != http.StatusOK {
+ t.Fatalf("exchange after the issuer resolved returned %d %q, want 200", code, body)
+ }
+}
diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.go
index 37ccaeb3..81262125 100644
--- a/internal/controlplane/sts.go
+++ b/internal/controlplane/sts.go
@@ -859,7 +859,7 @@ func (s *Server) mintDelegatedBiscuitFromJWT(
claims, token, err := identity.VerifyJWT(verifyCtx, subjectJWT, s.config.AllowedAudiences, s.getProviders())
if err != nil {
- return nil, time.Time{}, "", nil, http.StatusUnauthorized, fmt.Errorf("JWT validation failed: %w", err)
+ return nil, time.Time{}, "", nil, http.StatusUnauthorized, errors.New(s.jwtRefusalMessage(err))
}
if !allowWorkload && s.isWorkloadClaims(claims) {
return nil, time.Time{}, "", nil, http.StatusForbidden, errWorkloadIdentity
diff --git a/internal/identity/oidc.go b/internal/identity/oidc.go
index 7d38ab1e..9831f508 100644
--- a/internal/identity/oidc.go
+++ b/internal/identity/oidc.go
@@ -23,6 +23,17 @@ import (
"github.com/golang-jwt/jwt/v5"
)
+// UnknownIssuerError is returned by VerifyJWT when the token names an issuer
+// that has no resolved provider: either one that was never configured, or one
+// whose OpenID discovery has not succeeded yet. The caller tells the two apart.
+type UnknownIssuerError struct {
+ Issuer string
+}
+
+func (e *UnknownIssuerError) Error() string {
+ return fmt.Sprintf("unknown issuer: %s", e.Issuer)
+}
+
// VerifyJWT parses and cryptographically validates a JWT token against a list of allowed audiences
// and resolved OIDC providers.
func VerifyJWT(ctx context.Context, jwtStr string, allowedAudiences []string, providers map[string]*oidc.Provider) (jwt.MapClaims, *oidc.IDToken, error) {
@@ -37,7 +48,7 @@ func VerifyJWT(ctx context.Context, jwtStr string, allowedAudiences []string, pr
provider, ok := providers[iss]
if !ok {
- return nil, nil, fmt.Errorf("unknown issuer: %s", iss)
+ return nil, nil, &UnknownIssuerError{Issuer: iss}
}
// SkipClientIDCheck because the verifier only knows how to match a single
diff --git a/site/content/docs/reference/agentmesh-one.md b/site/content/docs/reference/agentmesh-one.md
index a58e9464..30387d15 100644
--- a/site/content/docs/reference/agentmesh-one.md
+++ b/site/content/docs/reference/agentmesh-one.md
@@ -43,7 +43,7 @@ downloaded tunnel connector.
| `--admin-token-path` | | File containing the admin token. Can also be set with `AGENTMESH_ADMIN_TOKEN`. Generated and saved to the data directory when neither is set. |
| `--policy-file` | | Protojson `PolicyConfig` that seeds the mesh policy on first boot. Ignored once the database has a policy. Without it, first boot seeds an open development policy and logs a warning. |
| `--issuer` | | External OIDC issuer(s), comma-separated. Optional. Without an issuer, enrollment works by token only. |
-| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`). Automatically added to `--issuer` and refused at `/user/*` and `/oauth/authorize`. |
+| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`). Automatically added to `--issuer` and refused at `/user/*` and `/oauth/authorize`. One that does not answer at start-up is resolved in the background and its tokens refused until then (`agentmesh_control_plane_workload_issuer_ready{issuer}`); a `--issuer` that does not answer is fatal. |
| `--oidc-client-id` | first audience | Client ID advertised on `/info`. |
| `--allowed-audiences` | `agentmesh-audience` | Accepted OIDC audiences. |
| `--enroll-qr` | when stdout is a terminal | Print a device-enrollment QR code at startup. Only for `https` URLs. |
diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md
index c35086d3..faef6fa6 100644
--- a/site/content/docs/reference/control-plane.md
+++ b/site/content/docs/reference/control-plane.md
@@ -20,7 +20,7 @@ agentmesh-control-plane admin unban --peer <id> lift a ban
| Flag | Default | Meaning |
|---|---|---|
| `--issuer` | | OIDC issuer URL(s), comma-separated. At least one of `--issuer` or `--workload-issuer` is required. The first issuer is advertised to enrolling nodes on `/info`. |
-| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`, such as `https://accounts.google.com=.gserviceaccount.com`). Automatically added to `--issuer`. Workload tokens can enroll, refresh, and exchange credentials (`/register`, `/refresh`, `/token/exchange`), and are refused at `/user/*` and `/oauth/authorize`. |
+| `--workload-issuer` | | Workload OIDC issuer(s), comma-separated (`<issuer>` or `<issuer>=<email-suffix>`, such as `https://accounts.google.com=.gserviceaccount.com`). Automatically added to `--issuer`. Workload tokens can enroll, refresh, and exchange credentials (`/register`, `/refresh`, `/token/exchange`), and are refused at `/user/*` and `/oauth/authorize`. A workload issuer that does not answer its discovery document at start-up does not stop the control plane: it keeps retrying in the background (backoff up to 5 minutes), refuses tokens from that issuer with `401` and a message naming it until it does, and reports it as `agentmesh_control_plane_workload_issuer_ready{issuer}` = 0. An `--issuer` that does not answer is still fatal. |
| `--allowed-audiences` | `agentmesh-audience` | Audiences accepted in OIDC tokens, comma-separated. |
| `--oidc-client-id` | first audience | OAuth client ID advertised on `/info`, for providers where it differs from the audience. |
| `--insecure-skip-tls-verify` | `false` | Skip TLS verification when fetching issuer metadata and keys. For a cluster issuer served with the cluster CA, or a local development issuer. |