Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Recover degraded database features without restarting the pod (#6901)

r33drichards154b6b7111b7
diff --git a/libs/fleet/backend/handlers/features.go b/libs/fleet/backend/handlers/features.go
new file mode 100644
index 0000000000..faf107af00
--- /dev/null
+++ b/libs/fleet/backend/handlers/features.go
@@ -0,0 +1,115 @@
+package handlers
+
+import (
+ "context"
+ "sync/atomic"
+
+ "cyclops-cs-backend/auth"
+ "cyclops-cs-backend/githubtrust"
+)
+
+// Features holds the database-backed dependencies that startup installs, in a
+// form a background goroutine can replace while the server is running.
+//
+// Handlers cannot hold these as plain fields if they are ever to change after
+// startup: setupRouter takes handlers.Handlers BY VALUE, and every route
+// handler and fact provider it registers captures that copy. Assigning to a
+// field afterwards updates nobody. Handlers therefore holds a *Features shared
+// by every copy, and the dependencies inside are swapped atomically so a retry
+// cannot race a request reading them.
+//
+// A nil *Features behaves like one with nothing installed, so Handlers{} stays
+// usable in tests that do not exercise these routes.
+type Features struct {
+ stateQuery atomic.Pointer[stateQuerySlot]
+ trustStore atomic.Pointer[trustStoreSlot]
+}
+
+// The slots exist because atomic.Pointer needs a concrete type. Storing the
+// interfaces directly in an atomic.Value would panic the moment two different
+// concrete implementations were stored, which is exactly what a retry does.
+type stateQuerySlot struct{ executor StateQueryExecutor }
+
+type trustStoreSlot struct{ store githubtrust.Store }
+
+func NewFeatures() *Features {
+ return &Features{}
+}
+
+// FeaturesWith is the constructor tests use to inject fakes; either argument
+// may be nil.
+func FeaturesWith(executor StateQueryExecutor, store githubtrust.Store) *Features {
+ features := NewFeatures()
+ features.SetStateQuery(executor)
+ features.SetTrustStore(store)
+ return features
+}
+
+func (f *Features) SetStateQuery(executor StateQueryExecutor) {
+ if f == nil {
+ return
+ }
+ f.stateQuery.Store(&stateQuerySlot{executor: executor})
+}
+
+func (f *Features) StateQuery() StateQueryExecutor {
+ if f == nil {
+ return nil
+ }
+ slot := f.stateQuery.Load()
+ if slot == nil {
+ return nil
+ }
+ return slot.executor
+}
+
+func (f *Features) SetTrustStore(store githubtrust.Store) {
+ if f == nil {
+ return
+ }
+ f.trustStore.Store(&trustStoreSlot{store: store})
+}
+
+func (f *Features) TrustStore() githubtrust.Store {
+ if f == nil {
+ return nil
+ }
+ slot := f.trustStore.Load()
+ if slot == nil {
+ return nil
+ }
+ return slot.store
+}
+
+// NewGitHubTrustResolverFor returns a resolver bound to the registry rather
+// than to one store instance, so auth's resolver global is written exactly once
+// at startup and never again. Rewriting that global from a retry goroutine
+// would race every request reading it in auth.EvaluateGitHubTrust.
+//
+// The resolver is always non-nil; it reports the store's absence per request
+// instead, which is what a not-yet-initialized database looks like.
+//
+// That changes one status code. Previously an uninitialized store left auth's
+// resolver nil and EvaluateGitHubTrust failed with "github trust resolver not
+// configured", which auth/middlewares.go answers with 401 "auth token is
+// invalid". Returning ErrDatabaseUnavailable instead routes it through
+// IsDatabaseUnavailable to 503 "authentication unavailable". 503 is the honest
+// answer — the token was never judged, the database was simply missing — and it
+// tells a GitHub OIDC client to retry, which now succeeds on its own once the
+// retry loop installs the store. A 401 would fail a CI job permanently for a
+// condition that clears itself.
+func NewGitHubTrustResolverFor(features *Features) auth.GitHubTrustResolver {
+ return registryTrustResolver{features: features}
+}
+
+type registryTrustResolver struct {
+ features *Features
+}
+
+func (r registryTrustResolver) ResolveGitHubTrustPolicies(ctx context.Context, repository string) ([]auth.GitHubTrustPolicy, error) {
+ store := r.features.TrustStore()
+ if store == nil {
+ return nil, auth.ErrDatabaseUnavailable
+ }
+ return githubTrustResolver{store: store}.ResolveGitHubTrustPolicies(ctx, repository)
+}
diff --git a/libs/fleet/backend/handlers/features_test.go b/libs/fleet/backend/handlers/features_test.go
new file mode 100644
index 0000000000..fc80acb69d
--- /dev/null
+++ b/libs/fleet/backend/handlers/features_test.go
@@ -0,0 +1,50 @@
+package handlers
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "cyclops-cs-backend/auth"
+)
+
+// The trust resolver is installed into an unguarded package global in auth that
+// every request reads, so it is bound to the registry once and never rewritten.
+// It therefore has to pick up a store installed after the fact.
+func TestTrustResolverSeesAStoreInstalledLater(t *testing.T) {
+ features := NewFeatures()
+ resolver := NewGitHubTrustResolverFor(features)
+
+ if _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud"); !errors.Is(err, auth.ErrDatabaseUnavailable) {
+ t.Fatalf("error with no store = %v, want ErrDatabaseUnavailable", err)
+ }
+
+ features.SetTrustStore(&fakeGitHubTrustStore{})
+
+ if _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud"); errors.Is(err, auth.ErrDatabaseUnavailable) {
+ t.Fatal("resolver still reports the database unavailable after a store was installed")
+ }
+}
+
+func TestFeaturesAreNilSafe(t *testing.T) {
+ var features *Features
+ if features.StateQuery() != nil || features.TrustStore() != nil {
+ t.Fatal("nil registry should read as empty")
+ }
+ features.SetStateQuery(nil) // must not panic
+}
+
+// An uninitialized store must read as "database unavailable", not as a generic
+// failure: auth/middlewares.go turns the former into 503 and everything else
+// into 401, and a GitHub OIDC client hitting a pod whose database has not come
+// up yet should be told to retry rather than that its token is invalid. The
+// retry loop makes that retry succeed on its own.
+func TestTrustResolverReportsAnAbsentStoreAsDatabaseUnavailable(t *testing.T) {
+ resolver := NewGitHubTrustResolverFor(NewFeatures())
+
+ _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud")
+
+ if !auth.IsDatabaseUnavailable(err) {
+ t.Fatalf("IsDatabaseUnavailable(%v) = false, want true so the caller answers 503 rather than 401", err)
+ }
+}
diff --git a/libs/fleet/backend/handlers/github_trust_policies.go b/libs/fleet/backend/handlers/github_trust_policies.go
index d393cc9871..fda7f4ff4b 100644
--- a/libs/fleet/backend/handlers/github_trust_policies.go
+++ b/libs/fleet/backend/handlers/github_trust_policies.go
@@ -43,13 +43,13 @@ func (h Handlers) ListGitHubTrustPolicies(w http.ResponseWriter, r *http.Request
writeErr(w, http.StatusUnauthorized, "missing user")
return
}
- if h.GitHubTrustPolicies == nil {
+ if h.Features.TrustStore() == nil {
writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")
return
}
ctx, cancel := databaseContext(r.Context())
defer cancel()
- policies, err := h.GitHubTrustPolicies.List(ctx, user.ID)
+ policies, err := h.Features.TrustStore().List(ctx, user.ID)
if err != nil {
writeGitHubTrustStoreErr(w, err, "failed to list github trust policies")
return
@@ -71,7 +71,7 @@ func (h Handlers) CreateGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
writeErr(w, http.StatusUnauthorized, "missing user")
return
}
- if h.GitHubTrustPolicies == nil {
+ if h.Features.TrustStore() == nil {
writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")
return
}
@@ -88,7 +88,7 @@ func (h Handlers) CreateGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
policy.OwnerSub = user.ID
ctx, cancel := databaseContext(r.Context())
defer cancel()
- if err := h.GitHubTrustPolicies.Create(ctx, policy); err != nil {
+ if err := h.Features.TrustStore().Create(ctx, policy); err != nil {
writeGitHubTrustStoreErr(w, err, "failed to create github trust policy")
return
}
@@ -101,14 +101,14 @@ func (h Handlers) UpdateGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
writeErr(w, http.StatusUnauthorized, "missing user")
return
}
- if h.GitHubTrustPolicies == nil {
+ if h.Features.TrustStore() == nil {
writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")
return
}
id := r.PathValue("id")
ctx, cancel := databaseContext(r.Context())
defer cancel()
- current, err := h.GitHubTrustPolicies.Get(ctx, user.ID, id)
+ current, err := h.Features.TrustStore().Get(ctx, user.ID, id)
if err != nil {
writeGitHubTrustStoreErr(w, err, "failed to load github trust policy")
return
@@ -148,7 +148,7 @@ func (h Handlers) UpdateGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
policy.ID = current.ID
policy.OwnerSub = current.OwnerSub
policy.CreatedAt = current.CreatedAt
- if err := h.GitHubTrustPolicies.Update(ctx, policy); err != nil {
+ if err := h.Features.TrustStore().Update(ctx, policy); err != nil {
if errors.Is(err, githubtrust.ErrNotFound) {
writeErr(w, http.StatusNotFound, "github trust policy not found")
return
@@ -165,13 +165,13 @@ func (h Handlers) DeleteGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
writeErr(w, http.StatusUnauthorized, "missing user")
return
}
- if h.GitHubTrustPolicies == nil {
+ if h.Features.TrustStore() == nil {
writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")
return
}
ctx, cancel := databaseContext(r.Context())
defer cancel()
- found, err := h.GitHubTrustPolicies.Delete(ctx, user.ID, r.PathValue("id"))
+ found, err := h.Features.TrustStore().Delete(ctx, user.ID, r.PathValue("id"))
if err != nil {
writeGitHubTrustStoreErr(w, err, "failed to delete github trust policy")
return
@@ -183,6 +183,10 @@ func (h Handlers) DeleteGitHubTrustPolicy(w http.ResponseWriter, r *http.Request
w.WriteHeader(http.StatusNoContent)
}
+// NewGitHubTrustResolver binds a resolver to one store. Production wiring uses
+// NewGitHubTrustResolverFor instead, so the store can be installed after
+// startup; this remains for tests that exercise the resolver against a fixed
+// store.
func NewGitHubTrustResolver(store githubtrust.Store) auth.GitHubTrustResolver {
if store == nil {
return nil
diff --git a/libs/fleet/backend/handlers/github_trust_policies_test.go b/libs/fleet/backend/handlers/github_trust_policies_test.go
index a363cad456..6a70df263d 100644
--- a/libs/fleet/backend/handlers/github_trust_policies_test.go
+++ b/libs/fleet/backend/handlers/github_trust_policies_test.go
@@ -122,7 +122,7 @@ func TestListGitHubTrustPolicies_Success(t *testing.T) {
}},
}
- h := Handlers{GitHubTrustPolicies: store, AuthCfg: authConfigForHandlers()}
+ h := Handlers{Features: FeaturesWith(nil, store), AuthCfg: authConfigForHandlers()}
r := httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil)
r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})
w := httptest.NewRecorder()
@@ -150,7 +150,7 @@ func TestListGitHubTrustPolicies_Success(t *testing.T) {
func TestCreateGitHubTrustPolicy_ValidatesAndPersists(t *testing.T) {
store := &fakeGitHubTrustStore{}
- h := Handlers{GitHubTrustPolicies: store}
+ h := Handlers{Features: FeaturesWith(nil, store)}
body := `{"name":"ci","repository":"trycua/cloud","allowed_namespaces":["ns-b","ns-a","ns-a"],"enabled":true}`
r := httptest.NewRequest(http.MethodPost, "/api/github-trust-policies", bytes.NewBufferString(body))
r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})
@@ -174,7 +174,7 @@ func TestCreateGitHubTrustPolicy_ValidatesAndPersists(t *testing.T) {
func TestPatchGitHubTrustPolicy_NotFound(t *testing.T) {
store := &fakeGitHubTrustStore{}
- h := Handlers{GitHubTrustPolicies: store}
+ h := Handlers{Features: FeaturesWith(nil, store)}
r := httptest.NewRequest(http.MethodPatch, "/api/github-trust-policies/missing", bytes.NewBufferString(`{"enabled":false}`))
r.SetPathValue("id", "missing")
r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})
@@ -189,7 +189,7 @@ func TestPatchGitHubTrustPolicy_NotFound(t *testing.T) {
func TestDeleteGitHubTrustPolicy_Success(t *testing.T) {
store := &fakeGitHubTrustStore{deleteFound: true}
- h := Handlers{GitHubTrustPolicies: store}
+ h := Handlers{Features: FeaturesWith(nil, store)}
r := httptest.NewRequest(http.MethodDelete, "/api/github-trust-policies/policy-1", nil)
r.SetPathValue("id", "policy-1")
r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})
@@ -220,7 +220,7 @@ func TestGitHubTrustPolicies_DisabledStore(t *testing.T) {
func TestGitHubTrustPolicies_StoreError(t *testing.T) {
store := &fakeGitHubTrustStore{listErr: errors.New("redis down")}
- h := Handlers{GitHubTrustPolicies: store}
+ h := Handlers{Features: FeaturesWith(nil, store)}
r := httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil)
r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})
w := httptest.NewRecorder()
@@ -272,7 +272,7 @@ func TestGitHubTrustPoliciesUseBoundedDatabaseContext(t *testing.T) {
response := httptest.NewRecorder()
start := time.Now()
- test.call(Handlers{GitHubTrustPolicies: store}, response, request)
+ test.call(Handlers{Features: FeaturesWith(nil, store)}, response, request)
assertBoundedDatabaseContext(t, test.context(store), start)
if test.name == "update" {
@@ -389,7 +389,7 @@ func TestGitHubTrustPolicies_DatabaseFailuresReturnServiceUnavailable(t *testing
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
store := &fakeGitHubTrustStore{listErr: test.err}
- h := Handlers{GitHubTrustPolicies: store}
+ h := Handlers{Features: FeaturesWith(nil, store)}
request := withUser(httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), &auth.User{ID: "user-123"})
response := httptest.NewRecorder()
@@ -404,7 +404,7 @@ func TestGitHubTrustPolicies_DatabaseFailuresReturnServiceUnavailable(t *testing
func TestGitHubTrustPolicies_PostgresErrorsRemainInternalServerErrors(t *testing.T) {
store := &fakeGitHubTrustStore{listErr: &pgconn.PgError{Code: "42501", Message: "permission denied"}}
- h := Handlers{GitHubTrustPolicies: store}
+ h := Handlers{Features: FeaturesWith(nil, store)}
request := withUser(httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), &auth.User{ID: "user-123"})
response := httptest.NewRecorder()
@@ -421,7 +421,7 @@ func TestUpdateGitHubTrustPolicySharesDatabaseBudget(t *testing.T) {
AllowedNamespaces: []string{"ns-a"}, Enabled: true,
}
store := &fakeGitHubTrustStore{getResult: policy}
- h := Handlers{GitHubTrustPolicies: store}
+ h := Handlers{Features: FeaturesWith(nil, store)}
request := withUser(httptest.NewRequest(http.MethodPatch, "/api/github-trust-policies/policy-1", bytes.NewBufferString(`{"enabled":false}`)), &auth.User{ID: "user-123"})
request.SetPathValue("id", "policy-1")
response := httptest.NewRecorder()
diff --git a/libs/fleet/backend/handlers/handlers.go b/libs/fleet/backend/handlers/handlers.go
index 7937890558..eb296e23f5 100644
--- a/libs/fleet/backend/handlers/handlers.go
+++ b/libs/fleet/backend/handlers/handlers.go
@@ -15,7 +15,6 @@ import (
"cyclops-cs-backend/auth"
"cyclops-cs-backend/chat"
"cyclops-cs-backend/config"
- "cyclops-cs-backend/githubtrust"
"cyclops-cs-backend/keycloak"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
@@ -35,9 +34,10 @@ type Handlers struct {
UserAccounts UserAccountService
WebhookVerifier WebhookVerifier
- GitHubTrustPolicies githubtrust.Store
-
- StateQueryExecutor StateQueryExecutor
+ // Features carries the database-backed dependencies (the state query
+ // executor and the GitHub trust policy store). It is a pointer because
+ // setupRouter copies Handlers by value; see features.go.
+ Features *Features
ChatAccess config.ChatAccessMode
Conversations chat.ConversationStore
@@ -57,6 +57,7 @@ type Handlers struct {
func New(admin *keycloak.Admin, cfg *config.Configuration) Handlers {
return Handlers{
+ Features: NewFeatures(),
Admin: admin,
UserAccounts: admin,
GatewayCfg: cfg.Gateway,
diff --git a/libs/fleet/backend/handlers/state_query.go b/libs/fleet/backend/handlers/state_query.go
index 1c8869c0c5..114b6c8f2a 100644
--- a/libs/fleet/backend/handlers/state_query.go
+++ b/libs/fleet/backend/handlers/state_query.go
@@ -87,7 +87,8 @@ func (writer *stateQueryStreamWriter) write(event any) error {
func (h Handlers) QueryState(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Accept-Query", "application/sql")
w.Header().Set("Cache-Control", "private, no-store")
- if h.StateQueryExecutor == nil {
+ executor := h.Features.StateQuery()
+ if executor == nil {
writeErr(w, http.StatusServiceUnavailable, "state query unavailable")
return
}
@@ -112,7 +113,7 @@ func (h Handlers) QueryState(w http.ResponseWriter, r *http.Request) {
stream := newStateQueryStreamWriter(w)
ctx, cancel := databaseContext(r.Context())
defer cancel()
- err = h.StateQueryExecutor.Execute(ctx, identity.PersonalGroup(r.Context(), user.ID), string(body), stream)
+ err = executor.Execute(ctx, identity.PersonalGroup(r.Context(), user.ID), string(body), stream)
if err != nil && !stream.started {
writeErr(w, http.StatusServiceUnavailable, "state query unavailable")
return
diff --git a/libs/fleet/backend/handlers/state_query_test.go b/libs/fleet/backend/handlers/state_query_test.go
index 7ba652cec0..30d8d1243e 100644
--- a/libs/fleet/backend/handlers/state_query_test.go
+++ b/libs/fleet/backend/handlers/state_query_test.go
@@ -70,7 +70,7 @@ func TestQueryStateStreamsRawQueryResults(t *testing.T) {
fields: []pgconn.FieldDescription{{Name: "name", DataTypeOID: 25}},
rows: [][]any{{"alice-ns"}, {"alice-pod"}},
}
- h := Handlers{StateQueryExecutor: executor}
+ h := Handlers{Features: FeaturesWith(executor, nil)}
query := "select pg_sleep(60)"
request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader(query)), &auth.User{ID: "alice"})
request.Header.Set("Content-Type", "application/sql; charset=utf-8")
@@ -99,7 +99,7 @@ func TestQueryStateStreamsRawQueryResults(t *testing.T) {
}
func TestQueryStateRequiresSQLContentType(t *testing.T) {
- h := Handlers{StateQueryExecutor: &fakeStateQueryExecutor{}}
+ h := Handlers{Features: FeaturesWith(&fakeStateQueryExecutor{}, nil)}
request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})
response := httptest.NewRecorder()
h.QueryState(response, request)
@@ -114,7 +114,7 @@ func TestQueryStateWritesExecutorErrorAsTerminalEvent(t *testing.T) {
rows: [][]any{{"alice-ns"}},
err: errors.New("stream failed"),
}
- h := Handlers{StateQueryExecutor: executor}
+ h := Handlers{Features: FeaturesWith(executor, nil)}
request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})
request.Header.Set("Content-Type", "application/sql")
response := httptest.NewRecorder()
@@ -147,7 +147,7 @@ func TestQueryStateUsesBoundedDatabaseContext(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
executor := &deadlineStateQueryExecutor{write: test.write, err: test.err}
- h := Handlers{StateQueryExecutor: executor}
+ h := Handlers{Features: FeaturesWith(executor, nil)}
request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})
request.Header.Set("Content-Type", "application/sql")
response := httptest.NewRecorder()
@@ -174,7 +174,7 @@ func TestQueryStateUsesBoundedDatabaseContext(t *testing.T) {
func TestQueryStateDatabaseFailureBeforeStreamReturnsServiceUnavailable(t *testing.T) {
executor := &deadlineStateQueryExecutor{err: auth.ErrDatabaseUnavailable}
- h := Handlers{StateQueryExecutor: executor}
+ h := Handlers{Features: FeaturesWith(executor, nil)}
request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})
request.Header.Set("Content-Type", "application/sql")
response := httptest.NewRecorder()
@@ -191,7 +191,7 @@ func TestQueryStateDatabaseFailureBeforeStreamReturnsServiceUnavailable(t *testi
func TestQueryStateDatabaseFailureAfterStreamWritesSanitizedTerminalEvent(t *testing.T) {
executor := &deadlineStateQueryExecutor{write: true, err: auth.ErrDatabaseUnavailable}
- h := Handlers{StateQueryExecutor: executor}
+ h := Handlers{Features: FeaturesWith(executor, nil)}
request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})
request.Header.Set("Content-Type", "application/sql")
response := httptest.NewRecorder()
@@ -222,7 +222,7 @@ func TestQueryStatePropagatesRequestCancellation(t *testing.T) {
started := make(chan struct{})
done := make(chan struct{})
handlerDone := make(chan struct{})
- h := Handlers{StateQueryExecutor: cancellationStateQueryExecutor{started: started, done: done}}
+ h := Handlers{Features: FeaturesWith(cancellationStateQueryExecutor{started: started, done: done}, nil)}
parent, cancel := context.WithCancel(context.Background())
defer cancel()
request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")).WithContext(parent), &auth.User{ID: "alice"})
diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.go
index 6f8e776f0d..e960785e18 100644
--- a/libs/fleet/backend/main.go
+++ b/libs/fleet/backend/main.go
@@ -334,7 +334,14 @@ func run() error {
h.WorkloadAdmin = workloadAdmin
h.WorkloadAudience = cfg.Keycloak.WorkloadAudience
h.WorkloadTokenURL = cfg.Keycloak.WorkloadTokenURL
- initializeDatabaseFeatures(ctx, cfg.Database, &h, defaultDatabaseFeatureDependencies())
+ // Install the trust resolver once, bound to the feature registry rather
+ // than to a store instance: auth keeps it in an unguarded package global
+ // that every request reads, so rewriting it from the retry goroutine below
+ // would be a data race.
+ auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolverFor(h.Features))
+ databaseFeatures := defaultDatabaseFeatureDependencies()
+ progress := initializeDatabaseFeatures(ctx, cfg.Database, &h, databaseFeatures)
+ go retryDatabaseFeatures(ctx, cfg.Database, &h, databaseFeatures, progress, databaseRetryInterval)
if cfg.Chat.Access.Enabled() {
h.Conversations = chat.NewMemoryConversationStore()
@@ -351,7 +358,11 @@ func run() error {
return nil
}
-const databaseStartupTimeout = 5 * time.Second
+const (
+ databaseStartupTimeout = 5 * time.Second
+ // How often a degraded pod re-attempts its database dependencies.
+ databaseRetryInterval = 30 * time.Second
+)
type databaseFeatureDependencies struct {
requireVersion func(context.Context, string, int64) error
@@ -369,34 +380,61 @@ func defaultDatabaseFeatureDependencies() databaseFeatureDependencies {
}
}
-func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies) {
- h.StateQueryExecutor = nil
- h.GitHubTrustPolicies = nil
- auth.SetGitHubTrustResolver(nil)
+// featureProgress records which database-backed dependencies are up, so a
+// retry only re-attempts what is still missing. Re-running the whole
+// initialization would clear a working state query executor every time it
+// retried a broken application database.
+type featureProgress struct {
+ stateQuery bool
+ applicationDB bool
+}
+
+func (p featureProgress) ready() bool { return p.stateQuery && p.applicationDB }
+
+// initializeDatabaseFeatures is the startup path: it clears anything stale and
+// installs whatever is configured, reporting what came up.
+func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies) featureProgress {
+ h.Features.SetStateQuery(nil)
+ h.Features.SetTrustStore(nil)
+
+ return attemptDatabaseFeatures(parent, cfg, h, dependencies, featureProgress{})
+}
+// attemptDatabaseFeatures installs whatever is configured and not already up,
+// leaving working dependencies untouched so it is safe to call repeatedly.
+func attemptDatabaseFeatures(parent context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies, progress featureProgress) featureProgress {
// The state query executor has its own DSN and its own failure path: it is
// initialized before the DATABASE_URL short-circuit below and a failure
// here does not stop the rest of this function, so it needs its own gauge.
// Reporting it through DatabaseFeaturesReady would claim a healthy
// /api/state/query whenever the application database came up fine.
- if cfg.StateQueryDSN != "" && cfg.StateQueryTenantPassword != "" {
+ switch {
+ case cfg.StateQueryDSN == "" || cfg.StateQueryTenantPassword == "":
+ metrics.StateQueryReady.WithLabelValues("false").Set(1)
+ progress.stateQuery = true
+ case progress.stateQuery:
+ // Already up; leave it alone.
+ default:
executor, err := dependencies.newStateQueryExecutor(cfg.StateQueryDSN, cfg.StateQueryTenantPassword)
if err != nil {
slog.Error("kubernetes state query: executor init failed; /api/state/query will return 503", "err", err)
metrics.StateQueryReady.WithLabelValues("true").Set(0)
} else {
- h.StateQueryExecutor = executor
+ h.Features.SetStateQuery(executor)
metrics.StateQueryReady.WithLabelValues("true").Set(1)
+ progress.stateQuery = true
slog.Info("kubernetes state query: query endpoint enabled")
}
- } else {
- metrics.StateQueryReady.WithLabelValues("false").Set(1)
}
if cfg.URL == "" {
slog.Info("postgres-backed features disabled (DATABASE_URL unset)")
metrics.DatabaseFeaturesReady.WithLabelValues("false").Set(1)
- return
+ progress.applicationDB = true
+ return progress
+ }
+ if progress.applicationDB {
+ return progress
}
// Every path below reports through this gauge. Readiness stays
@@ -404,25 +442,63 @@ func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfi
// operator learns that a serving pod came up degraded.
metrics.DatabaseFeaturesReady.WithLabelValues("true").Set(0)
+ // Deliberately the same budget on the retry path: a database that cannot
+ // answer within databaseStartupTimeout is still down as far as this pod is
+ // concerned, and a longer budget would only stretch the tick.
ctx, cancel := context.WithTimeout(parent, databaseStartupTimeout)
defer cancel()
if err := dependencies.requireVersion(ctx, cfg.URL, 1); err != nil {
slog.Error("postgres database schema unavailable; database-backed routes will return 503", "err", err)
- return
+ return progress
}
slog.Info("postgres database schema ready", "version", 1)
store, err := dependencies.newGitHubTrustStore(ctx, cfg.URL)
if err != nil {
slog.Error("github trust policies: init failed; /api/github-trust-policies will return 503", "err", err)
- return
+ return progress
}
if store != nil {
- h.GitHubTrustPolicies = store
- auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(store))
+ h.Features.SetTrustStore(store)
slog.Info("github trust policies: enabled")
}
metrics.DatabaseFeaturesReady.WithLabelValues("true").Set(1)
+ progress.applicationDB = true
+ return progress
+}
+
+// retryDatabaseFeatures re-attempts the dependencies that did not come up at
+// startup until they do, then stops. Without it a pod that started during a
+// database outage stays degraded until someone restarts it: initialization runs
+// once, and the serving tier deliberately keeps the pod Ready throughout, so
+// nothing else would ever notice.
+//
+// This is only reachable because the dependencies live behind handlers.Features
+// rather than in the Handlers copy the router captured.
+//
+// ctx is run()'s context.Background(), which nothing cancels today, so in
+// production this goroutine ends with the process. The ctx.Done() case is for
+// tests, and for the day run() grows a real shutdown context.
+func retryDatabaseFeatures(ctx context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies, progress featureProgress, interval time.Duration) {
+ if progress.ready() {
+ return
+ }
+ slog.Warn("database features degraded at startup; retrying in the background", "interval", interval)
+
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ progress = attemptDatabaseFeatures(ctx, cfg, h, dependencies, progress)
+ if progress.ready() {
+ slog.Info("database features recovered without a restart")
+ return
+ }
+ }
+ }
}
diff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.go
index 1955fac091..3a3563b670 100644
--- a/libs/fleet/backend/main_test.go
+++ b/libs/fleet/backend/main_test.go
@@ -660,8 +660,7 @@ func TestInitializeDatabaseFeatures(t *testing.T) {
},
}
h := handlers.Handlers{
- StateQueryExecutor: stateQueryExecutorStub{},
- GitHubTrustPolicies: githubTrustStoreStub{},
+ Features: handlers.FeaturesWith(stateQueryExecutorStub{}, githubTrustStoreStub{}),
}
auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(githubTrustStoreStub{}))
t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })
@@ -674,10 +673,10 @@ func TestInitializeDatabaseFeatures(t *testing.T) {
if got := newStoreCalls == 1; got != test.wantNewStore {
t.Fatalf("GitHub trust store constructor called = %t, want %t", got, test.wantNewStore)
}
- if got := h.StateQueryExecutor != nil; got != test.wantExecutor {
+ if got := h.Features.StateQuery() != nil; got != test.wantExecutor {
t.Fatalf("StateQueryExecutor present = %t, want %t", got, test.wantExecutor)
}
- if got := h.GitHubTrustPolicies != nil; got != test.wantStore {
+ if got := h.Features.TrustStore() != nil; got != test.wantStore {
t.Fatalf("GitHubTrustPolicies present = %t, want %t", got, test.wantStore)
}
assertStartupContext(t, requireVersionContext, test.wantRequireVersion)
@@ -818,7 +817,7 @@ func TestInitializeDatabaseFeaturesReportsReadinessMetric(t *testing.T) {
return githubTrustStoreStub{}, nil
},
}
- h := handlers.Handlers{}
+ h := handlers.Handlers{Features: handlers.NewFeatures()}
t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })
initializeDatabaseFeatures(context.Background(), test.config, &h, dependencies)
@@ -847,3 +846,135 @@ func gaugeValue(t *testing.T, gauge prometheus.Gauge) float64 {
}
return measurement.GetGauge().GetValue()
}
+
+// The point of the retry: a pod that started during a database outage recovers
+// on its own. Before this, initialization ran once and the pod stayed degraded
+// until someone restarted it.
+func TestRetryDatabaseFeaturesRecoversWithoutARestart(t *testing.T) {
+ metrics.DatabaseFeaturesReady.Reset()
+ attempts := 0
+ dependencies := databaseFeatureDependencies{
+ requireVersion: func(context.Context, string, int64) error {
+ attempts++
+ if attempts < 3 {
+ return fmt.Errorf("schema unavailable")
+ }
+ return nil
+ },
+ newStateQueryExecutor: func(string, string) (handlers.StateQueryExecutor, error) {
+ return stateQueryExecutorStub{}, nil
+ },
+ newGitHubTrustStore: func(context.Context, string) (githubtrust.Store, error) {
+ return githubTrustStoreStub{}, nil
+ },
+ }
+ cfg := config.DatabaseConfiguration{URL: "postgres://application/application"}
+ h := handlers.Handlers{Features: handlers.NewFeatures()}
+ t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })
+
+ progress := initializeDatabaseFeatures(context.Background(), cfg, &h, dependencies)
+ if progress.ready() {
+ t.Fatal("startup reported ready despite the schema being unavailable")
+ }
+ if got := gaugeValue(t, metrics.DatabaseFeaturesReady.WithLabelValues("true")); got != 0 {
+ t.Fatalf("gauge after failed startup = %v, want 0", got)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ retryDatabaseFeatures(ctx, cfg, &h, dependencies, progress, time.Millisecond)
+
+ if h.Features.TrustStore() == nil {
+ t.Fatal("trust store still absent after the retry succeeded")
+ }
+ if got := gaugeValue(t, metrics.DatabaseFeaturesReady.WithLabelValues("true")); got != 1 {
+ t.Fatalf("gauge after recovery = %v, want 1 so the alert resolves", got)
+ }
+}
+
+// A retry driven by a broken application database must not disturb a state
+// query executor that came up fine — re-running the whole initialization would
+// clear it on every tick.
+func TestRetryDatabaseFeaturesLeavesWorkingDependenciesAlone(t *testing.T) {
+ installed := stateQueryExecutorStub{}
+ executorCalls := 0
+ dependencies := databaseFeatureDependencies{
+ requireVersion: func(context.Context, string, int64) error {
+ return fmt.Errorf("schema unavailable")
+ },
+ newStateQueryExecutor: func(string, string) (handlers.StateQueryExecutor, error) {
+ executorCalls++
+ return installed, nil
+ },
+ newGitHubTrustStore: func(context.Context, string) (githubtrust.Store, error) {
+ return githubTrustStoreStub{}, nil
+ },
+ }
+ cfg := config.DatabaseConfiguration{
+ URL: "postgres://application/application",
+ StateQueryDSN: "postgres://state-query/state-query",
+ StateQueryTenantPassword: "tenant-password",
+ }
+ h := handlers.Handlers{Features: handlers.NewFeatures()}
+ t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })
+
+ progress := initializeDatabaseFeatures(context.Background(), cfg, &h, dependencies)
+ for range 3 {
+ progress = attemptDatabaseFeatures(context.Background(), cfg, &h, dependencies, progress)
+ if h.Features.StateQuery() == nil {
+ t.Fatal("a retry cleared the working state query executor")
+ }
+ }
+ if executorCalls != 1 {
+ t.Fatalf("state query constructed %d times, want 1 — retries must skip what is already up", executorCalls)
+ }
+}
+
+// The reason the dependencies live behind Features at all: setupRouter copies
+// Handlers by value, so a later install has to be visible through an existing
+// copy or the retry is pointless.
+func TestFeaturesInstallIsVisibleThroughAnExistingHandlersCopy(t *testing.T) {
+ h := handlers.Handlers{Features: handlers.NewFeatures()}
+ captured := h // what setupRouter and every route handler hold
+
+ h.Features.SetTrustStore(githubTrustStoreStub{})
+
+ if captured.Features.TrustStore() == nil {
+ t.Fatal("install not visible through the captured copy; the retry would update nobody")
+ }
+}
+
+// The loop must stop once everything is up, or a recovered pod carries a ticker
+// and a goroutine for the rest of its life.
+func TestRetryDatabaseFeaturesStopsOnceReady(t *testing.T) {
+ attempts := 0
+ dependencies := databaseFeatureDependencies{
+ requireVersion: func(context.Context, string, int64) error {
+ attempts++
+ if attempts < 2 {
+ return fmt.Errorf("schema unavailable")
+ }
+ return nil
+ },
+ newStateQueryExecutor: func(string, string) (handlers.StateQueryExecutor, error) {
+ return stateQueryExecutorStub{}, nil
+ },
+ newGitHubTrustStore: func(context.Context, string) (githubtrust.Store, error) {
+ return githubTrustStoreStub{}, nil
+ },
+ }
+ cfg := config.DatabaseConfiguration{URL: "postgres://application/application"}
+ h := handlers.Handlers{Features: handlers.NewFeatures()}
+ t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })
+
+ progress := initializeDatabaseFeatures(context.Background(), cfg, &h, dependencies)
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ retryDatabaseFeatures(ctx, cfg, &h, dependencies, progress, time.Millisecond)
+
+ settled := attempts
+ time.Sleep(50 * time.Millisecond) // many ticks' worth at a 1ms interval
+ if attempts != settled {
+ t.Fatalf("attempts kept climbing after recovery (%d -> %d); the loop did not stop", settled, attempts)
+ }
+}