Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane: retry OIDC discovery with backoff on startup
Antonio Ojeac8be317e05dc
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex 07d0dda7..e1093241 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -198,7 +198,7 @@ func (s *Server) discoverProviders() error {Transport: tr,}providerCtx := oidc.ClientContext(s.ctx, client)- provider, err := oidc.NewProvider(providerCtx, iss)+ provider, err := discoverProviderWithRetry(providerCtx, iss, oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay)if err != nil {return fmt.Errorf("failed to create provider for %s: %w", iss, err)}@@ -207,6 +207,40 @@ func (s *Server) discoverProviders() error {return nil}+// Defaults for discoverProviderWithRetry; kept small enough that a real outage still+// surfaces quickly (worst case ~15s) while riding out a transient hiccup during rollouts.+const (+ oidcDiscoveryMaxAttempts = 5+ oidcDiscoveryBaseDelay = 1 * time.Second+ oidcDiscoveryMaxDelay = 8 * time.Second+)++// discoverProviderWithRetry retries OIDC discovery with exponential backoff so a transient+// upstream hiccup (e.g. the identity provider mid-rollout) doesn't crash the control plane.+func discoverProviderWithRetry(ctx context.Context, issuer string, maxAttempts int, baseDelay, maxDelay time.Duration) (*oidc.Provider, error) {+ var lastErr error+ for attempt := 0; attempt < maxAttempts; attempt++ {+ if attempt > 0 {+ delay := baseDelay * time.Duration(1<<uint(attempt-1))+ if delay > maxDelay {+ delay = maxDelay+ }+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(delay):+ }+ }+ provider, err := oidc.NewProvider(ctx, issuer)+ if err == nil {+ return provider, nil+ }+ lastErr = err+ logger.Warnf("OIDC discovery attempt %d/%d for %s failed: %v", attempt+1, maxAttempts, issuer, err)+ }+ return nil, lastErr+}+func (s *Server) getProviders() map[string]*oidc.Provider {s.providersMu.RLock()defer s.providersMu.RUnlock()diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.goindex fb3bc7c5..81a23d39 100644--- a/internal/controlplane/server_test.go+++ b/internal/controlplane/server_test.go@@ -29,6 +29,7 @@ import ("os""path/filepath""reflect"+ "sync/atomic""testing""time"@@ -1199,3 +1200,46 @@ func TestResolveRolesAndRoleImpersonationProtection(t *testing.T) {}})}++func TestDiscoverProviderWithRetry(t *testing.T) {+ t.Run("succeeds after transient failures", func(t *testing.T) {+ var attempts int32+ mux := http.NewServeMux()+ srv := httptest.NewServer(mux)+ defer srv.Close()+ issuer := srv.URL++ mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {+ if atomic.AddInt32(&attempts, 1) < 3 {+ http.Error(w, "upstream connect error", http.StatusServiceUnavailable)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "issuer": issuer,+ "jwks_uri": issuer + "/keys",+ })+ })++ if _, err := discoverProviderWithRetry(context.Background(), issuer, 5, time.Millisecond, 5*time.Millisecond); err != nil {+ t.Fatalf("expected discovery to eventually succeed, got: %v", err)+ }+ if got := atomic.LoadInt32(&attempts); got != 3 {+ t.Errorf("expected 3 attempts, got %d", got)+ }+ })++ t.Run("gives up after max attempts", func(t *testing.T) {+ mux := http.NewServeMux()+ srv := httptest.NewServer(mux)+ defer srv.Close()++ mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "always down", http.StatusServiceUnavailable)+ })++ if _, err := discoverProviderWithRetry(context.Background(), srv.URL, 3, time.Millisecond, 5*time.Millisecond); err == nil {+ t.Fatal("expected discovery to fail after exhausting retries")+ }+ })+}