Skip to content

Agent Mesh

controlplane: retry OIDC discovery with backoff on startup

Antonio Ojeac8be317e05dc
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 07d0dda7..e1093241 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -198,7 +198,7 @@ func (s *Server) discoverProviders() error {
Transport: tr,
}
providerCtx := oidc.ClientContext(s.ctx, client)
- provider, err := oidc.NewProvider(providerCtx, iss)
+ provider, err := discoverProviderWithRetry(providerCtx, iss, oidcDiscoveryMaxAttempts, oidcDiscoveryBaseDelay, oidcDiscoveryMaxDelay)
if err != nil {
return fmt.Errorf("failed to create provider for %s: %w", iss, err)
}
@@ -207,6 +207,40 @@ func (s *Server) discoverProviders() error {
return nil
}
+// Defaults for discoverProviderWithRetry; kept small enough that a real outage still
+// surfaces quickly (worst case ~15s) while riding out a transient hiccup during rollouts.
+const (
+ oidcDiscoveryMaxAttempts = 5
+ oidcDiscoveryBaseDelay = 1 * time.Second
+ oidcDiscoveryMaxDelay = 8 * time.Second
+)
+
+// discoverProviderWithRetry retries OIDC discovery with exponential backoff so a transient
+// upstream hiccup (e.g. the identity provider mid-rollout) doesn't crash the control plane.
+func discoverProviderWithRetry(ctx context.Context, issuer string, maxAttempts int, baseDelay, maxDelay time.Duration) (*oidc.Provider, error) {
+ var lastErr error
+ for attempt := 0; attempt < maxAttempts; attempt++ {
+ if attempt > 0 {
+ delay := baseDelay * time.Duration(1<<uint(attempt-1))
+ if delay > maxDelay {
+ delay = maxDelay
+ }
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(delay):
+ }
+ }
+ provider, err := oidc.NewProvider(ctx, issuer)
+ if err == nil {
+ return provider, nil
+ }
+ lastErr = err
+ logger.Warnf("OIDC discovery attempt %d/%d for %s failed: %v", attempt+1, maxAttempts, issuer, err)
+ }
+ return nil, lastErr
+}
+
func (s *Server) getProviders() map[string]*oidc.Provider {
s.providersMu.RLock()
defer s.providersMu.RUnlock()
diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.go
index fb3bc7c5..81a23d39 100644
--- a/internal/controlplane/server_test.go
+++ b/internal/controlplane/server_test.go
@@ -29,6 +29,7 @@ import (
"os"
"path/filepath"
"reflect"
+ "sync/atomic"
"testing"
"time"
@@ -1199,3 +1200,46 @@ func TestResolveRolesAndRoleImpersonationProtection(t *testing.T) {
}
})
}
+
+func TestDiscoverProviderWithRetry(t *testing.T) {
+ t.Run("succeeds after transient failures", func(t *testing.T) {
+ var attempts int32
+ mux := http.NewServeMux()
+ srv := httptest.NewServer(mux)
+ defer srv.Close()
+ issuer := srv.URL
+
+ mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
+ if atomic.AddInt32(&attempts, 1) < 3 {
+ http.Error(w, "upstream connect error", http.StatusServiceUnavailable)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "issuer": issuer,
+ "jwks_uri": issuer + "/keys",
+ })
+ })
+
+ if _, err := discoverProviderWithRetry(context.Background(), issuer, 5, time.Millisecond, 5*time.Millisecond); err != nil {
+ t.Fatalf("expected discovery to eventually succeed, got: %v", err)
+ }
+ if got := atomic.LoadInt32(&attempts); got != 3 {
+ t.Errorf("expected 3 attempts, got %d", got)
+ }
+ })
+
+ t.Run("gives up after max attempts", func(t *testing.T) {
+ mux := http.NewServeMux()
+ srv := httptest.NewServer(mux)
+ defer srv.Close()
+
+ mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "always down", http.StatusServiceUnavailable)
+ })
+
+ if _, err := discoverProviderWithRetry(context.Background(), srv.URL, 3, time.Millisecond, 5*time.Millisecond); err == nil {
+ t.Fatal("expected discovery to fail after exhausting retries")
+ }
+ })
+}