Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Report the state query executor's startup health separately (#6887)
r33drichardse3d822e8abf3
diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.goindex 17f1834100..6f8e776f0d 100644--- a/libs/fleet/backend/main.go+++ b/libs/fleet/backend/main.go@@ -374,14 +374,23 @@ func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfih.GitHubTrustPolicies = nilauth.SetGitHubTrustResolver(nil)+ // The state query executor has its own DSN and its own failure path: it is+ // initialized before the DATABASE_URL short-circuit below and a failure+ // here does not stop the rest of this function, so it needs its own gauge.+ // Reporting it through DatabaseFeaturesReady would claim a healthy+ // /api/state/query whenever the application database came up fine.if cfg.StateQueryDSN != "" && cfg.StateQueryTenantPassword != "" {executor, err := dependencies.newStateQueryExecutor(cfg.StateQueryDSN, cfg.StateQueryTenantPassword)if err != nil {slog.Error("kubernetes state query: executor init failed; /api/state/query will return 503", "err", err)+ metrics.StateQueryReady.WithLabelValues("true").Set(0)} else {h.StateQueryExecutor = executor+ metrics.StateQueryReady.WithLabelValues("true").Set(1)slog.Info("kubernetes state query: query endpoint enabled")}+ } else {+ metrics.StateQueryReady.WithLabelValues("false").Set(1)}if cfg.URL == "" {diff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.goindex aee640ea2c..1955fac091 100644--- a/libs/fleet/backend/main_test.go+++ b/libs/fleet/backend/main_test.go@@ -742,20 +742,40 @@ func TestInitializeDatabaseFeaturesReportsReadinessMetric(t *testing.T) {config config.DatabaseConfigurationrequireVersionError errornewStoreError error+ newExecutorError errorwantConfigured stringwantValue float64+ wantStateQuery string+ wantStateQueryValue float64}{{- name: "unset database url reports the disabled configuration as ready",- config: config.DatabaseConfiguration{},- wantConfigured: "false",- wantValue: 1,+ name: "unset database url reports the disabled configuration as ready",+ config: config.DatabaseConfiguration{},+ wantConfigured: "false",+ wantValue: 1,+ wantStateQuery: "false",+ wantStateQueryValue: 1,+ },+ {+ name: "state query failure is reported independently of the application database",+ config: config.DatabaseConfiguration{+ URL: "postgres://application/application",+ StateQueryDSN: "postgres://state-query/state-query",+ StateQueryTenantPassword: "tenant-password",+ },+ newExecutorError: fmt.Errorf("invalid state query configuration"),+ wantConfigured: "true",+ wantValue: 1,+ wantStateQuery: "true",+ wantStateQueryValue: 0,},{- name: "healthy database reports ready",- config: config.DatabaseConfiguration{URL: "postgres://application/application"},- wantConfigured: "true",- wantValue: 1,+ name: "healthy database reports ready",+ config: config.DatabaseConfiguration{URL: "postgres://application/application"},+ wantConfigured: "true",+ wantValue: 1,+ wantStateQuery: "false",+ wantStateQueryValue: 1,},{name: "unavailable schema reports degraded",@@ -763,24 +783,32 @@ func TestInitializeDatabaseFeaturesReportsReadinessMetric(t *testing.T) {requireVersionError: fmt.Errorf("schema unavailable"),wantConfigured: "true",wantValue: 0,+ wantStateQuery: "false",+ wantStateQueryValue: 1,},{- name: "trust store failure reports degraded",- config: config.DatabaseConfiguration{URL: "postgres://application/application"},- newStoreError: fmt.Errorf("store unavailable"),- wantConfigured: "true",- wantValue: 0,+ name: "trust store failure reports degraded",+ config: config.DatabaseConfiguration{URL: "postgres://application/application"},+ newStoreError: fmt.Errorf("store unavailable"),+ wantConfigured: "true",+ wantValue: 0,+ wantStateQuery: "false",+ wantStateQueryValue: 1,},}for _, test := range tests {t.Run(test.name, func(t *testing.T) {metrics.DatabaseFeaturesReady.Reset()+ metrics.StateQueryReady.Reset()dependencies := databaseFeatureDependencies{requireVersion: func(context.Context, string, int64) error {return test.requireVersionError},newStateQueryExecutor: func(string, string) (handlers.StateQueryExecutor, error) {+ if test.newExecutorError != nil {+ return nil, test.newExecutorError+ }return stateQueryExecutorStub{}, nil},newGitHubTrustStore: func(context.Context, string) (githubtrust.Store, error) {@@ -800,6 +828,13 @@ func TestInitializeDatabaseFeaturesReportsReadinessMetric(t *testing.T) {t.Fatalf("cyclops_cs_database_features_ready{configured=%q} = %v, want %v",test.wantConfigured, got, test.wantValue)}+ if test.wantStateQuery != "" {+ gotStateQuery := gaugeValue(t, metrics.StateQueryReady.WithLabelValues(test.wantStateQuery))+ if gotStateQuery != test.wantStateQueryValue {+ t.Fatalf("cyclops_cs_state_query_ready{configured=%q} = %v, want %v",+ test.wantStateQuery, gotStateQuery, test.wantStateQueryValue)+ }+ }})}}diff --git a/libs/fleet/backend/metrics/metrics.go b/libs/fleet/backend/metrics/metrics.goindex 3b77fdc138..f081f24d3c 100644--- a/libs/fleet/backend/metrics/metrics.go+++ b/libs/fleet/backend/metrics/metrics.go@@ -85,9 +85,24 @@ var (//// Initialization runs once at startup, so the value is fixed for the// lifetime of the process and recovery means restarting the pod.+ // This gauge tracks the APPLICATION database (DATABASE_URL) only, which+ // is what the GitHub trust policy routes need. The Kubernetes state query+ // executor behind /api/state/query is a separate dependency on a separate+ // DSN (STATE_QUERY_DSN) and is reported by StateQueryReady below —+ // initializeDatabaseFeatures logs and continues when it fails, so a broken+ // state query alongside a healthy application database would otherwise+ // leave this gauge reporting 1.DatabaseFeaturesReady = promauto.NewGaugeVec(prometheus.GaugeOpts{Name: "cyclops_cs_database_features_ready",- Help: "1 when the PostgreSQL-backed features initialized at startup (or are intentionally disabled), 0 when a configured database failed to come up.",+ Help: "1 when the application database features initialized at startup (or are intentionally disabled), 0 when a configured database failed to come up.",+ }, []string{"configured"})++ // StateQueryReady is the same signal for the /api/state/query executor,+ // which has its own DSN and its own failure path. configured="false"+ // (STATE_QUERY_DSN or its tenant password unset) always reports 1.+ StateQueryReady = promauto.NewGaugeVec(prometheus.GaugeOpts{+ Name: "cyclops_cs_state_query_ready",+ Help: "1 when the Kubernetes state query executor initialized at startup (or is intentionally disabled), 0 when a configured executor failed to come up.",}, []string{"configured"})UpstreamProxyDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{