Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Recover degraded database features without restarting the pod (#6901)
r33drichards154b6b7111b7
diff --git a/libs/fleet/backend/handlers/features.go b/libs/fleet/backend/handlers/features.gonew file mode 100644index 0000000000..faf107af00--- /dev/null+++ b/libs/fleet/backend/handlers/features.go@@ -0,0 +1,115 @@+package handlers++import (+ "context"+ "sync/atomic"++ "cyclops-cs-backend/auth"+ "cyclops-cs-backend/githubtrust"+)++// Features holds the database-backed dependencies that startup installs, in a+// form a background goroutine can replace while the server is running.+//+// Handlers cannot hold these as plain fields if they are ever to change after+// startup: setupRouter takes handlers.Handlers BY VALUE, and every route+// handler and fact provider it registers captures that copy. Assigning to a+// field afterwards updates nobody. Handlers therefore holds a *Features shared+// by every copy, and the dependencies inside are swapped atomically so a retry+// cannot race a request reading them.+//+// A nil *Features behaves like one with nothing installed, so Handlers{} stays+// usable in tests that do not exercise these routes.+type Features struct {+ stateQuery atomic.Pointer[stateQuerySlot]+ trustStore atomic.Pointer[trustStoreSlot]+}++// The slots exist because atomic.Pointer needs a concrete type. Storing the+// interfaces directly in an atomic.Value would panic the moment two different+// concrete implementations were stored, which is exactly what a retry does.+type stateQuerySlot struct{ executor StateQueryExecutor }++type trustStoreSlot struct{ store githubtrust.Store }++func NewFeatures() *Features {+ return &Features{}+}++// FeaturesWith is the constructor tests use to inject fakes; either argument+// may be nil.+func FeaturesWith(executor StateQueryExecutor, store githubtrust.Store) *Features {+ features := NewFeatures()+ features.SetStateQuery(executor)+ features.SetTrustStore(store)+ return features+}++func (f *Features) SetStateQuery(executor StateQueryExecutor) {+ if f == nil {+ return+ }+ f.stateQuery.Store(&stateQuerySlot{executor: executor})+}++func (f *Features) StateQuery() StateQueryExecutor {+ if f == nil {+ return nil+ }+ slot := f.stateQuery.Load()+ if slot == nil {+ return nil+ }+ return slot.executor+}++func (f *Features) SetTrustStore(store githubtrust.Store) {+ if f == nil {+ return+ }+ f.trustStore.Store(&trustStoreSlot{store: store})+}++func (f *Features) TrustStore() githubtrust.Store {+ if f == nil {+ return nil+ }+ slot := f.trustStore.Load()+ if slot == nil {+ return nil+ }+ return slot.store+}++// NewGitHubTrustResolverFor returns a resolver bound to the registry rather+// than to one store instance, so auth's resolver global is written exactly once+// at startup and never again. Rewriting that global from a retry goroutine+// would race every request reading it in auth.EvaluateGitHubTrust.+//+// The resolver is always non-nil; it reports the store's absence per request+// instead, which is what a not-yet-initialized database looks like.+//+// That changes one status code. Previously an uninitialized store left auth's+// resolver nil and EvaluateGitHubTrust failed with "github trust resolver not+// configured", which auth/middlewares.go answers with 401 "auth token is+// invalid". Returning ErrDatabaseUnavailable instead routes it through+// IsDatabaseUnavailable to 503 "authentication unavailable". 503 is the honest+// answer — the token was never judged, the database was simply missing — and it+// tells a GitHub OIDC client to retry, which now succeeds on its own once the+// retry loop installs the store. A 401 would fail a CI job permanently for a+// condition that clears itself.+func NewGitHubTrustResolverFor(features *Features) auth.GitHubTrustResolver {+ return registryTrustResolver{features: features}+}++type registryTrustResolver struct {+ features *Features+}++func (r registryTrustResolver) ResolveGitHubTrustPolicies(ctx context.Context, repository string) ([]auth.GitHubTrustPolicy, error) {+ store := r.features.TrustStore()+ if store == nil {+ return nil, auth.ErrDatabaseUnavailable+ }+ return githubTrustResolver{store: store}.ResolveGitHubTrustPolicies(ctx, repository)+}diff --git a/libs/fleet/backend/handlers/features_test.go b/libs/fleet/backend/handlers/features_test.gonew file mode 100644index 0000000000..fc80acb69d--- /dev/null+++ b/libs/fleet/backend/handlers/features_test.go@@ -0,0 +1,50 @@+package handlers++import (+ "context"+ "errors"+ "testing"++ "cyclops-cs-backend/auth"+)++// The trust resolver is installed into an unguarded package global in auth that+// every request reads, so it is bound to the registry once and never rewritten.+// It therefore has to pick up a store installed after the fact.+func TestTrustResolverSeesAStoreInstalledLater(t *testing.T) {+ features := NewFeatures()+ resolver := NewGitHubTrustResolverFor(features)++ if _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud"); !errors.Is(err, auth.ErrDatabaseUnavailable) {+ t.Fatalf("error with no store = %v, want ErrDatabaseUnavailable", err)+ }++ features.SetTrustStore(&fakeGitHubTrustStore{})++ if _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud"); errors.Is(err, auth.ErrDatabaseUnavailable) {+ t.Fatal("resolver still reports the database unavailable after a store was installed")+ }+}++func TestFeaturesAreNilSafe(t *testing.T) {+ var features *Features+ if features.StateQuery() != nil || features.TrustStore() != nil {+ t.Fatal("nil registry should read as empty")+ }+ features.SetStateQuery(nil) // must not panic+}++// An uninitialized store must read as "database unavailable", not as a generic+// failure: auth/middlewares.go turns the former into 503 and everything else+// into 401, and a GitHub OIDC client hitting a pod whose database has not come+// up yet should be told to retry rather than that its token is invalid. The+// retry loop makes that retry succeed on its own.+func TestTrustResolverReportsAnAbsentStoreAsDatabaseUnavailable(t *testing.T) {+ resolver := NewGitHubTrustResolverFor(NewFeatures())++ _, err := resolver.ResolveGitHubTrustPolicies(context.Background(), "trycua/cloud")++ if !auth.IsDatabaseUnavailable(err) {+ t.Fatalf("IsDatabaseUnavailable(%v) = false, want true so the caller answers 503 rather than 401", err)+ }+}diff --git a/libs/fleet/backend/handlers/github_trust_policies.go b/libs/fleet/backend/handlers/github_trust_policies.goindex d393cc9871..fda7f4ff4b 100644--- a/libs/fleet/backend/handlers/github_trust_policies.go+++ b/libs/fleet/backend/handlers/github_trust_policies.go@@ -43,13 +43,13 @@ func (h Handlers) ListGitHubTrustPolicies(w http.ResponseWriter, r *http.RequestwriteErr(w, http.StatusUnauthorized, "missing user")return}- if h.GitHubTrustPolicies == nil {+ if h.Features.TrustStore() == nil {writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")return}ctx, cancel := databaseContext(r.Context())defer cancel()- policies, err := h.GitHubTrustPolicies.List(ctx, user.ID)+ policies, err := h.Features.TrustStore().List(ctx, user.ID)if err != nil {writeGitHubTrustStoreErr(w, err, "failed to list github trust policies")return@@ -71,7 +71,7 @@ func (h Handlers) CreateGitHubTrustPolicy(w http.ResponseWriter, r *http.RequestwriteErr(w, http.StatusUnauthorized, "missing user")return}- if h.GitHubTrustPolicies == nil {+ if h.Features.TrustStore() == nil {writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")return}@@ -88,7 +88,7 @@ func (h Handlers) CreateGitHubTrustPolicy(w http.ResponseWriter, r *http.Requestpolicy.OwnerSub = user.IDctx, cancel := databaseContext(r.Context())defer cancel()- if err := h.GitHubTrustPolicies.Create(ctx, policy); err != nil {+ if err := h.Features.TrustStore().Create(ctx, policy); err != nil {writeGitHubTrustStoreErr(w, err, "failed to create github trust policy")return}@@ -101,14 +101,14 @@ func (h Handlers) UpdateGitHubTrustPolicy(w http.ResponseWriter, r *http.RequestwriteErr(w, http.StatusUnauthorized, "missing user")return}- if h.GitHubTrustPolicies == nil {+ if h.Features.TrustStore() == nil {writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")return}id := r.PathValue("id")ctx, cancel := databaseContext(r.Context())defer cancel()- current, err := h.GitHubTrustPolicies.Get(ctx, user.ID, id)+ current, err := h.Features.TrustStore().Get(ctx, user.ID, id)if err != nil {writeGitHubTrustStoreErr(w, err, "failed to load github trust policy")return@@ -148,7 +148,7 @@ func (h Handlers) UpdateGitHubTrustPolicy(w http.ResponseWriter, r *http.Requestpolicy.ID = current.IDpolicy.OwnerSub = current.OwnerSubpolicy.CreatedAt = current.CreatedAt- if err := h.GitHubTrustPolicies.Update(ctx, policy); err != nil {+ if err := h.Features.TrustStore().Update(ctx, policy); err != nil {if errors.Is(err, githubtrust.ErrNotFound) {writeErr(w, http.StatusNotFound, "github trust policy not found")return@@ -165,13 +165,13 @@ func (h Handlers) DeleteGitHubTrustPolicy(w http.ResponseWriter, r *http.RequestwriteErr(w, http.StatusUnauthorized, "missing user")return}- if h.GitHubTrustPolicies == nil {+ if h.Features.TrustStore() == nil {writeErr(w, http.StatusServiceUnavailable, "github trust policies are not configured")return}ctx, cancel := databaseContext(r.Context())defer cancel()- found, err := h.GitHubTrustPolicies.Delete(ctx, user.ID, r.PathValue("id"))+ found, err := h.Features.TrustStore().Delete(ctx, user.ID, r.PathValue("id"))if err != nil {writeGitHubTrustStoreErr(w, err, "failed to delete github trust policy")return@@ -183,6 +183,10 @@ func (h Handlers) DeleteGitHubTrustPolicy(w http.ResponseWriter, r *http.Requestw.WriteHeader(http.StatusNoContent)}+// NewGitHubTrustResolver binds a resolver to one store. Production wiring uses+// NewGitHubTrustResolverFor instead, so the store can be installed after+// startup; this remains for tests that exercise the resolver against a fixed+// store.func NewGitHubTrustResolver(store githubtrust.Store) auth.GitHubTrustResolver {if store == nil {return nildiff --git a/libs/fleet/backend/handlers/github_trust_policies_test.go b/libs/fleet/backend/handlers/github_trust_policies_test.goindex a363cad456..6a70df263d 100644--- a/libs/fleet/backend/handlers/github_trust_policies_test.go+++ b/libs/fleet/backend/handlers/github_trust_policies_test.go@@ -122,7 +122,7 @@ func TestListGitHubTrustPolicies_Success(t *testing.T) {}},}- h := Handlers{GitHubTrustPolicies: store, AuthCfg: authConfigForHandlers()}+ h := Handlers{Features: FeaturesWith(nil, store), AuthCfg: authConfigForHandlers()}r := httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil)r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})w := httptest.NewRecorder()@@ -150,7 +150,7 @@ func TestListGitHubTrustPolicies_Success(t *testing.T) {func TestCreateGitHubTrustPolicy_ValidatesAndPersists(t *testing.T) {store := &fakeGitHubTrustStore{}- h := Handlers{GitHubTrustPolicies: store}+ h := Handlers{Features: FeaturesWith(nil, store)}body := `{"name":"ci","repository":"trycua/cloud","allowed_namespaces":["ns-b","ns-a","ns-a"],"enabled":true}`r := httptest.NewRequest(http.MethodPost, "/api/github-trust-policies", bytes.NewBufferString(body))r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})@@ -174,7 +174,7 @@ func TestCreateGitHubTrustPolicy_ValidatesAndPersists(t *testing.T) {func TestPatchGitHubTrustPolicy_NotFound(t *testing.T) {store := &fakeGitHubTrustStore{}- h := Handlers{GitHubTrustPolicies: store}+ h := Handlers{Features: FeaturesWith(nil, store)}r := httptest.NewRequest(http.MethodPatch, "/api/github-trust-policies/missing", bytes.NewBufferString(`{"enabled":false}`))r.SetPathValue("id", "missing")r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})@@ -189,7 +189,7 @@ func TestPatchGitHubTrustPolicy_NotFound(t *testing.T) {func TestDeleteGitHubTrustPolicy_Success(t *testing.T) {store := &fakeGitHubTrustStore{deleteFound: true}- h := Handlers{GitHubTrustPolicies: store}+ h := Handlers{Features: FeaturesWith(nil, store)}r := httptest.NewRequest(http.MethodDelete, "/api/github-trust-policies/policy-1", nil)r.SetPathValue("id", "policy-1")r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})@@ -220,7 +220,7 @@ func TestGitHubTrustPolicies_DisabledStore(t *testing.T) {func TestGitHubTrustPolicies_StoreError(t *testing.T) {store := &fakeGitHubTrustStore{listErr: errors.New("redis down")}- h := Handlers{GitHubTrustPolicies: store}+ h := Handlers{Features: FeaturesWith(nil, store)}r := httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil)r = withUser(r, &auth.User{ID: "user-123", AZP: "cyclops-cs-spa"})w := httptest.NewRecorder()@@ -272,7 +272,7 @@ func TestGitHubTrustPoliciesUseBoundedDatabaseContext(t *testing.T) {response := httptest.NewRecorder()start := time.Now()- test.call(Handlers{GitHubTrustPolicies: store}, response, request)+ test.call(Handlers{Features: FeaturesWith(nil, store)}, response, request)assertBoundedDatabaseContext(t, test.context(store), start)if test.name == "update" {@@ -389,7 +389,7 @@ func TestGitHubTrustPolicies_DatabaseFailuresReturnServiceUnavailable(t *testingfor _, test := range tests {t.Run(test.name, func(t *testing.T) {store := &fakeGitHubTrustStore{listErr: test.err}- h := Handlers{GitHubTrustPolicies: store}+ h := Handlers{Features: FeaturesWith(nil, store)}request := withUser(httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), &auth.User{ID: "user-123"})response := httptest.NewRecorder()@@ -404,7 +404,7 @@ func TestGitHubTrustPolicies_DatabaseFailuresReturnServiceUnavailable(t *testingfunc TestGitHubTrustPolicies_PostgresErrorsRemainInternalServerErrors(t *testing.T) {store := &fakeGitHubTrustStore{listErr: &pgconn.PgError{Code: "42501", Message: "permission denied"}}- h := Handlers{GitHubTrustPolicies: store}+ h := Handlers{Features: FeaturesWith(nil, store)}request := withUser(httptest.NewRequest(http.MethodGet, "/api/github-trust-policies", nil), &auth.User{ID: "user-123"})response := httptest.NewRecorder()@@ -421,7 +421,7 @@ func TestUpdateGitHubTrustPolicySharesDatabaseBudget(t *testing.T) {AllowedNamespaces: []string{"ns-a"}, Enabled: true,}store := &fakeGitHubTrustStore{getResult: policy}- h := Handlers{GitHubTrustPolicies: store}+ h := Handlers{Features: FeaturesWith(nil, store)}request := withUser(httptest.NewRequest(http.MethodPatch, "/api/github-trust-policies/policy-1", bytes.NewBufferString(`{"enabled":false}`)), &auth.User{ID: "user-123"})request.SetPathValue("id", "policy-1")response := httptest.NewRecorder()diff --git a/libs/fleet/backend/handlers/handlers.go b/libs/fleet/backend/handlers/handlers.goindex 7937890558..eb296e23f5 100644--- a/libs/fleet/backend/handlers/handlers.go+++ b/libs/fleet/backend/handlers/handlers.go@@ -15,7 +15,6 @@ import ("cyclops-cs-backend/auth""cyclops-cs-backend/chat""cyclops-cs-backend/config"- "cyclops-cs-backend/githubtrust""cyclops-cs-backend/keycloak""go.opentelemetry.io/otel""go.opentelemetry.io/otel/trace"@@ -35,9 +34,10 @@ type Handlers struct {UserAccounts UserAccountServiceWebhookVerifier WebhookVerifier- GitHubTrustPolicies githubtrust.Store-- StateQueryExecutor StateQueryExecutor+ // Features carries the database-backed dependencies (the state query+ // executor and the GitHub trust policy store). It is a pointer because+ // setupRouter copies Handlers by value; see features.go.+ Features *FeaturesChatAccess config.ChatAccessModeConversations chat.ConversationStore@@ -57,6 +57,7 @@ type Handlers struct {func New(admin *keycloak.Admin, cfg *config.Configuration) Handlers {return Handlers{+ Features: NewFeatures(),Admin: admin,UserAccounts: admin,GatewayCfg: cfg.Gateway,diff --git a/libs/fleet/backend/handlers/state_query.go b/libs/fleet/backend/handlers/state_query.goindex 1c8869c0c5..114b6c8f2a 100644--- a/libs/fleet/backend/handlers/state_query.go+++ b/libs/fleet/backend/handlers/state_query.go@@ -87,7 +87,8 @@ func (writer *stateQueryStreamWriter) write(event any) error {func (h Handlers) QueryState(w http.ResponseWriter, r *http.Request) {w.Header().Set("Accept-Query", "application/sql")w.Header().Set("Cache-Control", "private, no-store")- if h.StateQueryExecutor == nil {+ executor := h.Features.StateQuery()+ if executor == nil {writeErr(w, http.StatusServiceUnavailable, "state query unavailable")return}@@ -112,7 +113,7 @@ func (h Handlers) QueryState(w http.ResponseWriter, r *http.Request) {stream := newStateQueryStreamWriter(w)ctx, cancel := databaseContext(r.Context())defer cancel()- err = h.StateQueryExecutor.Execute(ctx, identity.PersonalGroup(r.Context(), user.ID), string(body), stream)+ err = executor.Execute(ctx, identity.PersonalGroup(r.Context(), user.ID), string(body), stream)if err != nil && !stream.started {writeErr(w, http.StatusServiceUnavailable, "state query unavailable")returndiff --git a/libs/fleet/backend/handlers/state_query_test.go b/libs/fleet/backend/handlers/state_query_test.goindex 7ba652cec0..30d8d1243e 100644--- a/libs/fleet/backend/handlers/state_query_test.go+++ b/libs/fleet/backend/handlers/state_query_test.go@@ -70,7 +70,7 @@ func TestQueryStateStreamsRawQueryResults(t *testing.T) {fields: []pgconn.FieldDescription{{Name: "name", DataTypeOID: 25}},rows: [][]any{{"alice-ns"}, {"alice-pod"}},}- h := Handlers{StateQueryExecutor: executor}+ h := Handlers{Features: FeaturesWith(executor, nil)}query := "select pg_sleep(60)"request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader(query)), &auth.User{ID: "alice"})request.Header.Set("Content-Type", "application/sql; charset=utf-8")@@ -99,7 +99,7 @@ func TestQueryStateStreamsRawQueryResults(t *testing.T) {}func TestQueryStateRequiresSQLContentType(t *testing.T) {- h := Handlers{StateQueryExecutor: &fakeStateQueryExecutor{}}+ h := Handlers{Features: FeaturesWith(&fakeStateQueryExecutor{}, nil)}request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})response := httptest.NewRecorder()h.QueryState(response, request)@@ -114,7 +114,7 @@ func TestQueryStateWritesExecutorErrorAsTerminalEvent(t *testing.T) {rows: [][]any{{"alice-ns"}},err: errors.New("stream failed"),}- h := Handlers{StateQueryExecutor: executor}+ h := Handlers{Features: FeaturesWith(executor, nil)}request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})request.Header.Set("Content-Type", "application/sql")response := httptest.NewRecorder()@@ -147,7 +147,7 @@ func TestQueryStateUsesBoundedDatabaseContext(t *testing.T) {for _, test := range tests {t.Run(test.name, func(t *testing.T) {executor := &deadlineStateQueryExecutor{write: test.write, err: test.err}- h := Handlers{StateQueryExecutor: executor}+ h := Handlers{Features: FeaturesWith(executor, nil)}request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})request.Header.Set("Content-Type", "application/sql")response := httptest.NewRecorder()@@ -174,7 +174,7 @@ func TestQueryStateUsesBoundedDatabaseContext(t *testing.T) {func TestQueryStateDatabaseFailureBeforeStreamReturnsServiceUnavailable(t *testing.T) {executor := &deadlineStateQueryExecutor{err: auth.ErrDatabaseUnavailable}- h := Handlers{StateQueryExecutor: executor}+ h := Handlers{Features: FeaturesWith(executor, nil)}request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})request.Header.Set("Content-Type", "application/sql")response := httptest.NewRecorder()@@ -191,7 +191,7 @@ func TestQueryStateDatabaseFailureBeforeStreamReturnsServiceUnavailable(t *testifunc TestQueryStateDatabaseFailureAfterStreamWritesSanitizedTerminalEvent(t *testing.T) {executor := &deadlineStateQueryExecutor{write: true, err: auth.ErrDatabaseUnavailable}- h := Handlers{StateQueryExecutor: executor}+ h := Handlers{Features: FeaturesWith(executor, nil)}request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")), &auth.User{ID: "alice"})request.Header.Set("Content-Type", "application/sql")response := httptest.NewRecorder()@@ -222,7 +222,7 @@ func TestQueryStatePropagatesRequestCancellation(t *testing.T) {started := make(chan struct{})done := make(chan struct{})handlerDone := make(chan struct{})- h := Handlers{StateQueryExecutor: cancellationStateQueryExecutor{started: started, done: done}}+ h := Handlers{Features: FeaturesWith(cancellationStateQueryExecutor{started: started, done: done}, nil)}parent, cancel := context.WithCancel(context.Background())defer cancel()request := withUser(httptest.NewRequest("QUERY", "/api/state/query", strings.NewReader("select 1")).WithContext(parent), &auth.User{ID: "alice"})diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.goindex 6f8e776f0d..e960785e18 100644--- a/libs/fleet/backend/main.go+++ b/libs/fleet/backend/main.go@@ -334,7 +334,14 @@ func run() error {h.WorkloadAdmin = workloadAdminh.WorkloadAudience = cfg.Keycloak.WorkloadAudienceh.WorkloadTokenURL = cfg.Keycloak.WorkloadTokenURL- initializeDatabaseFeatures(ctx, cfg.Database, &h, defaultDatabaseFeatureDependencies())+ // Install the trust resolver once, bound to the feature registry rather+ // than to a store instance: auth keeps it in an unguarded package global+ // that every request reads, so rewriting it from the retry goroutine below+ // would be a data race.+ auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolverFor(h.Features))+ databaseFeatures := defaultDatabaseFeatureDependencies()+ progress := initializeDatabaseFeatures(ctx, cfg.Database, &h, databaseFeatures)+ go retryDatabaseFeatures(ctx, cfg.Database, &h, databaseFeatures, progress, databaseRetryInterval)if cfg.Chat.Access.Enabled() {h.Conversations = chat.NewMemoryConversationStore()@@ -351,7 +358,11 @@ func run() error {return nil}-const databaseStartupTimeout = 5 * time.Second+const (+ databaseStartupTimeout = 5 * time.Second+ // How often a degraded pod re-attempts its database dependencies.+ databaseRetryInterval = 30 * time.Second+)type databaseFeatureDependencies struct {requireVersion func(context.Context, string, int64) error@@ -369,34 +380,61 @@ func defaultDatabaseFeatureDependencies() databaseFeatureDependencies {}}-func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies) {- h.StateQueryExecutor = nil- h.GitHubTrustPolicies = nil- auth.SetGitHubTrustResolver(nil)+// featureProgress records which database-backed dependencies are up, so a+// retry only re-attempts what is still missing. Re-running the whole+// initialization would clear a working state query executor every time it+// retried a broken application database.+type featureProgress struct {+ stateQuery bool+ applicationDB bool+}++func (p featureProgress) ready() bool { return p.stateQuery && p.applicationDB }++// initializeDatabaseFeatures is the startup path: it clears anything stale and+// installs whatever is configured, reporting what came up.+func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies) featureProgress {+ h.Features.SetStateQuery(nil)+ h.Features.SetTrustStore(nil)++ return attemptDatabaseFeatures(parent, cfg, h, dependencies, featureProgress{})+}+// attemptDatabaseFeatures installs whatever is configured and not already up,+// leaving working dependencies untouched so it is safe to call repeatedly.+func attemptDatabaseFeatures(parent context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies, progress featureProgress) featureProgress {// The state query executor has its own DSN and its own failure path: it is// initialized before the DATABASE_URL short-circuit below and a failure// here does not stop the rest of this function, so it needs its own gauge.// Reporting it through DatabaseFeaturesReady would claim a healthy// /api/state/query whenever the application database came up fine.- if cfg.StateQueryDSN != "" && cfg.StateQueryTenantPassword != "" {+ switch {+ case cfg.StateQueryDSN == "" || cfg.StateQueryTenantPassword == "":+ metrics.StateQueryReady.WithLabelValues("false").Set(1)+ progress.stateQuery = true+ case progress.stateQuery:+ // Already up; leave it alone.+ default:executor, err := dependencies.newStateQueryExecutor(cfg.StateQueryDSN, cfg.StateQueryTenantPassword)if err != nil {slog.Error("kubernetes state query: executor init failed; /api/state/query will return 503", "err", err)metrics.StateQueryReady.WithLabelValues("true").Set(0)} else {- h.StateQueryExecutor = executor+ h.Features.SetStateQuery(executor)metrics.StateQueryReady.WithLabelValues("true").Set(1)+ progress.stateQuery = trueslog.Info("kubernetes state query: query endpoint enabled")}- } else {- metrics.StateQueryReady.WithLabelValues("false").Set(1)}if cfg.URL == "" {slog.Info("postgres-backed features disabled (DATABASE_URL unset)")metrics.DatabaseFeaturesReady.WithLabelValues("false").Set(1)- return+ progress.applicationDB = true+ return progress+ }+ if progress.applicationDB {+ return progress}// Every path below reports through this gauge. Readiness stays@@ -404,25 +442,63 @@ func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfi// operator learns that a serving pod came up degraded.metrics.DatabaseFeaturesReady.WithLabelValues("true").Set(0)+ // Deliberately the same budget on the retry path: a database that cannot+ // answer within databaseStartupTimeout is still down as far as this pod is+ // concerned, and a longer budget would only stretch the tick.ctx, cancel := context.WithTimeout(parent, databaseStartupTimeout)defer cancel()if err := dependencies.requireVersion(ctx, cfg.URL, 1); err != nil {slog.Error("postgres database schema unavailable; database-backed routes will return 503", "err", err)- return+ return progress}slog.Info("postgres database schema ready", "version", 1)store, err := dependencies.newGitHubTrustStore(ctx, cfg.URL)if err != nil {slog.Error("github trust policies: init failed; /api/github-trust-policies will return 503", "err", err)- return+ return progress}if store != nil {- h.GitHubTrustPolicies = store- auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(store))+ h.Features.SetTrustStore(store)slog.Info("github trust policies: enabled")}metrics.DatabaseFeaturesReady.WithLabelValues("true").Set(1)+ progress.applicationDB = true+ return progress+}++// retryDatabaseFeatures re-attempts the dependencies that did not come up at+// startup until they do, then stops. Without it a pod that started during a+// database outage stays degraded until someone restarts it: initialization runs+// once, and the serving tier deliberately keeps the pod Ready throughout, so+// nothing else would ever notice.+//+// This is only reachable because the dependencies live behind handlers.Features+// rather than in the Handlers copy the router captured.+//+// ctx is run()'s context.Background(), which nothing cancels today, so in+// production this goroutine ends with the process. The ctx.Done() case is for+// tests, and for the day run() grows a real shutdown context.+func retryDatabaseFeatures(ctx context.Context, cfg config.DatabaseConfiguration, h *handlers.Handlers, dependencies databaseFeatureDependencies, progress featureProgress, interval time.Duration) {+ if progress.ready() {+ return+ }+ slog.Warn("database features degraded at startup; retrying in the background", "interval", interval)++ ticker := time.NewTicker(interval)+ defer ticker.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ progress = attemptDatabaseFeatures(ctx, cfg, h, dependencies, progress)+ if progress.ready() {+ slog.Info("database features recovered without a restart")+ return+ }+ }+ }}diff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.goindex 1955fac091..3a3563b670 100644--- a/libs/fleet/backend/main_test.go+++ b/libs/fleet/backend/main_test.go@@ -660,8 +660,7 @@ func TestInitializeDatabaseFeatures(t *testing.T) {},}h := handlers.Handlers{- StateQueryExecutor: stateQueryExecutorStub{},- GitHubTrustPolicies: githubTrustStoreStub{},+ Features: handlers.FeaturesWith(stateQueryExecutorStub{}, githubTrustStoreStub{}),}auth.SetGitHubTrustResolver(handlers.NewGitHubTrustResolver(githubTrustStoreStub{}))t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })@@ -674,10 +673,10 @@ func TestInitializeDatabaseFeatures(t *testing.T) {if got := newStoreCalls == 1; got != test.wantNewStore {t.Fatalf("GitHub trust store constructor called = %t, want %t", got, test.wantNewStore)}- if got := h.StateQueryExecutor != nil; got != test.wantExecutor {+ if got := h.Features.StateQuery() != nil; got != test.wantExecutor {t.Fatalf("StateQueryExecutor present = %t, want %t", got, test.wantExecutor)}- if got := h.GitHubTrustPolicies != nil; got != test.wantStore {+ if got := h.Features.TrustStore() != nil; got != test.wantStore {t.Fatalf("GitHubTrustPolicies present = %t, want %t", got, test.wantStore)}assertStartupContext(t, requireVersionContext, test.wantRequireVersion)@@ -818,7 +817,7 @@ func TestInitializeDatabaseFeaturesReportsReadinessMetric(t *testing.T) {return githubTrustStoreStub{}, nil},}- h := handlers.Handlers{}+ h := handlers.Handlers{Features: handlers.NewFeatures()}t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })initializeDatabaseFeatures(context.Background(), test.config, &h, dependencies)@@ -847,3 +846,135 @@ func gaugeValue(t *testing.T, gauge prometheus.Gauge) float64 {}return measurement.GetGauge().GetValue()}++// The point of the retry: a pod that started during a database outage recovers+// on its own. Before this, initialization ran once and the pod stayed degraded+// until someone restarted it.+func TestRetryDatabaseFeaturesRecoversWithoutARestart(t *testing.T) {+ metrics.DatabaseFeaturesReady.Reset()+ attempts := 0+ dependencies := databaseFeatureDependencies{+ requireVersion: func(context.Context, string, int64) error {+ attempts+++ if attempts < 3 {+ return fmt.Errorf("schema unavailable")+ }+ return nil+ },+ newStateQueryExecutor: func(string, string) (handlers.StateQueryExecutor, error) {+ return stateQueryExecutorStub{}, nil+ },+ newGitHubTrustStore: func(context.Context, string) (githubtrust.Store, error) {+ return githubTrustStoreStub{}, nil+ },+ }+ cfg := config.DatabaseConfiguration{URL: "postgres://application/application"}+ h := handlers.Handlers{Features: handlers.NewFeatures()}+ t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })++ progress := initializeDatabaseFeatures(context.Background(), cfg, &h, dependencies)+ if progress.ready() {+ t.Fatal("startup reported ready despite the schema being unavailable")+ }+ if got := gaugeValue(t, metrics.DatabaseFeaturesReady.WithLabelValues("true")); got != 0 {+ t.Fatalf("gauge after failed startup = %v, want 0", got)+ }++ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ retryDatabaseFeatures(ctx, cfg, &h, dependencies, progress, time.Millisecond)++ if h.Features.TrustStore() == nil {+ t.Fatal("trust store still absent after the retry succeeded")+ }+ if got := gaugeValue(t, metrics.DatabaseFeaturesReady.WithLabelValues("true")); got != 1 {+ t.Fatalf("gauge after recovery = %v, want 1 so the alert resolves", got)+ }+}++// A retry driven by a broken application database must not disturb a state+// query executor that came up fine — re-running the whole initialization would+// clear it on every tick.+func TestRetryDatabaseFeaturesLeavesWorkingDependenciesAlone(t *testing.T) {+ installed := stateQueryExecutorStub{}+ executorCalls := 0+ dependencies := databaseFeatureDependencies{+ requireVersion: func(context.Context, string, int64) error {+ return fmt.Errorf("schema unavailable")+ },+ newStateQueryExecutor: func(string, string) (handlers.StateQueryExecutor, error) {+ executorCalls+++ return installed, nil+ },+ newGitHubTrustStore: func(context.Context, string) (githubtrust.Store, error) {+ return githubTrustStoreStub{}, nil+ },+ }+ cfg := config.DatabaseConfiguration{+ URL: "postgres://application/application",+ StateQueryDSN: "postgres://state-query/state-query",+ StateQueryTenantPassword: "tenant-password",+ }+ h := handlers.Handlers{Features: handlers.NewFeatures()}+ t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })++ progress := initializeDatabaseFeatures(context.Background(), cfg, &h, dependencies)+ for range 3 {+ progress = attemptDatabaseFeatures(context.Background(), cfg, &h, dependencies, progress)+ if h.Features.StateQuery() == nil {+ t.Fatal("a retry cleared the working state query executor")+ }+ }+ if executorCalls != 1 {+ t.Fatalf("state query constructed %d times, want 1 — retries must skip what is already up", executorCalls)+ }+}++// The reason the dependencies live behind Features at all: setupRouter copies+// Handlers by value, so a later install has to be visible through an existing+// copy or the retry is pointless.+func TestFeaturesInstallIsVisibleThroughAnExistingHandlersCopy(t *testing.T) {+ h := handlers.Handlers{Features: handlers.NewFeatures()}+ captured := h // what setupRouter and every route handler hold++ h.Features.SetTrustStore(githubTrustStoreStub{})++ if captured.Features.TrustStore() == nil {+ t.Fatal("install not visible through the captured copy; the retry would update nobody")+ }+}++// The loop must stop once everything is up, or a recovered pod carries a ticker+// and a goroutine for the rest of its life.+func TestRetryDatabaseFeaturesStopsOnceReady(t *testing.T) {+ attempts := 0+ dependencies := databaseFeatureDependencies{+ requireVersion: func(context.Context, string, int64) error {+ attempts+++ if attempts < 2 {+ return fmt.Errorf("schema unavailable")+ }+ return nil+ },+ newStateQueryExecutor: func(string, string) (handlers.StateQueryExecutor, error) {+ return stateQueryExecutorStub{}, nil+ },+ newGitHubTrustStore: func(context.Context, string) (githubtrust.Store, error) {+ return githubTrustStoreStub{}, nil+ },+ }+ cfg := config.DatabaseConfiguration{URL: "postgres://application/application"}+ h := handlers.Handlers{Features: handlers.NewFeatures()}+ t.Cleanup(func() { auth.SetGitHubTrustResolver(nil) })++ progress := initializeDatabaseFeatures(context.Background(), cfg, &h, dependencies)+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ retryDatabaseFeatures(ctx, cfg, &h, dependencies, progress, time.Millisecond)++ settled := attempts+ time.Sleep(50 * time.Millisecond) // many ticks' worth at a 1ms interval+ if attempts != settled {+ t.Fatalf("attempts kept climbing after recovery (%d -> %d); the loop did not stop", settled, attempts)+ }+}