Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Report the state query executor's startup health separately (#6887)

r33drichardse3d822e8abf3
diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.go
index 17f1834100..6f8e776f0d 100644
--- a/libs/fleet/backend/main.go
+++ b/libs/fleet/backend/main.go
@@ -374,14 +374,23 @@ func initializeDatabaseFeatures(parent context.Context, cfg config.DatabaseConfi
h.GitHubTrustPolicies = nil
auth.SetGitHubTrustResolver(nil)
+ // The state query executor has its own DSN and its own failure path: it is
+ // initialized before the DATABASE_URL short-circuit below and a failure
+ // here does not stop the rest of this function, so it needs its own gauge.
+ // Reporting it through DatabaseFeaturesReady would claim a healthy
+ // /api/state/query whenever the application database came up fine.
if cfg.StateQueryDSN != "" && cfg.StateQueryTenantPassword != "" {
executor, err := dependencies.newStateQueryExecutor(cfg.StateQueryDSN, cfg.StateQueryTenantPassword)
if err != nil {
slog.Error("kubernetes state query: executor init failed; /api/state/query will return 503", "err", err)
+ metrics.StateQueryReady.WithLabelValues("true").Set(0)
} else {
h.StateQueryExecutor = executor
+ metrics.StateQueryReady.WithLabelValues("true").Set(1)
slog.Info("kubernetes state query: query endpoint enabled")
}
+ } else {
+ metrics.StateQueryReady.WithLabelValues("false").Set(1)
}
if cfg.URL == "" {
diff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.go
index aee640ea2c..1955fac091 100644
--- a/libs/fleet/backend/main_test.go
+++ b/libs/fleet/backend/main_test.go
@@ -742,20 +742,40 @@ func TestInitializeDatabaseFeaturesReportsReadinessMetric(t *testing.T) {
config config.DatabaseConfiguration
requireVersionError error
newStoreError error
+ newExecutorError error
wantConfigured string
wantValue float64
+ wantStateQuery string
+ wantStateQueryValue float64
}{
{
- name: "unset database url reports the disabled configuration as ready",
- config: config.DatabaseConfiguration{},
- wantConfigured: "false",
- wantValue: 1,
+ name: "unset database url reports the disabled configuration as ready",
+ config: config.DatabaseConfiguration{},
+ wantConfigured: "false",
+ wantValue: 1,
+ wantStateQuery: "false",
+ wantStateQueryValue: 1,
+ },
+ {
+ name: "state query failure is reported independently of the application database",
+ config: config.DatabaseConfiguration{
+ URL: "postgres://application/application",
+ StateQueryDSN: "postgres://state-query/state-query",
+ StateQueryTenantPassword: "tenant-password",
+ },
+ newExecutorError: fmt.Errorf("invalid state query configuration"),
+ wantConfigured: "true",
+ wantValue: 1,
+ wantStateQuery: "true",
+ wantStateQueryValue: 0,
},
{
- name: "healthy database reports ready",
- config: config.DatabaseConfiguration{URL: "postgres://application/application"},
- wantConfigured: "true",
- wantValue: 1,
+ name: "healthy database reports ready",
+ config: config.DatabaseConfiguration{URL: "postgres://application/application"},
+ wantConfigured: "true",
+ wantValue: 1,
+ wantStateQuery: "false",
+ wantStateQueryValue: 1,
},
{
name: "unavailable schema reports degraded",
@@ -763,24 +783,32 @@ func TestInitializeDatabaseFeaturesReportsReadinessMetric(t *testing.T) {
requireVersionError: fmt.Errorf("schema unavailable"),
wantConfigured: "true",
wantValue: 0,
+ wantStateQuery: "false",
+ wantStateQueryValue: 1,
},
{
- name: "trust store failure reports degraded",
- config: config.DatabaseConfiguration{URL: "postgres://application/application"},
- newStoreError: fmt.Errorf("store unavailable"),
- wantConfigured: "true",
- wantValue: 0,
+ name: "trust store failure reports degraded",
+ config: config.DatabaseConfiguration{URL: "postgres://application/application"},
+ newStoreError: fmt.Errorf("store unavailable"),
+ wantConfigured: "true",
+ wantValue: 0,
+ wantStateQuery: "false",
+ wantStateQueryValue: 1,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
metrics.DatabaseFeaturesReady.Reset()
+ metrics.StateQueryReady.Reset()
dependencies := databaseFeatureDependencies{
requireVersion: func(context.Context, string, int64) error {
return test.requireVersionError
},
newStateQueryExecutor: func(string, string) (handlers.StateQueryExecutor, error) {
+ if test.newExecutorError != nil {
+ return nil, test.newExecutorError
+ }
return stateQueryExecutorStub{}, nil
},
newGitHubTrustStore: func(context.Context, string) (githubtrust.Store, error) {
@@ -800,6 +828,13 @@ func TestInitializeDatabaseFeaturesReportsReadinessMetric(t *testing.T) {
t.Fatalf("cyclops_cs_database_features_ready{configured=%q} = %v, want %v",
test.wantConfigured, got, test.wantValue)
}
+ if test.wantStateQuery != "" {
+ gotStateQuery := gaugeValue(t, metrics.StateQueryReady.WithLabelValues(test.wantStateQuery))
+ if gotStateQuery != test.wantStateQueryValue {
+ t.Fatalf("cyclops_cs_state_query_ready{configured=%q} = %v, want %v",
+ test.wantStateQuery, gotStateQuery, test.wantStateQueryValue)
+ }
+ }
})
}
}
diff --git a/libs/fleet/backend/metrics/metrics.go b/libs/fleet/backend/metrics/metrics.go
index 3b77fdc138..f081f24d3c 100644
--- a/libs/fleet/backend/metrics/metrics.go
+++ b/libs/fleet/backend/metrics/metrics.go
@@ -85,9 +85,24 @@ var (
//
// Initialization runs once at startup, so the value is fixed for the
// lifetime of the process and recovery means restarting the pod.
+ // This gauge tracks the APPLICATION database (DATABASE_URL) only, which
+ // is what the GitHub trust policy routes need. The Kubernetes state query
+ // executor behind /api/state/query is a separate dependency on a separate
+ // DSN (STATE_QUERY_DSN) and is reported by StateQueryReady below —
+ // initializeDatabaseFeatures logs and continues when it fails, so a broken
+ // state query alongside a healthy application database would otherwise
+ // leave this gauge reporting 1.
DatabaseFeaturesReady = promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "cyclops_cs_database_features_ready",
- Help: "1 when the PostgreSQL-backed features initialized at startup (or are intentionally disabled), 0 when a configured database failed to come up.",
+ Help: "1 when the application database features initialized at startup (or are intentionally disabled), 0 when a configured database failed to come up.",
+ }, []string{"configured"})
+
+ // StateQueryReady is the same signal for the /api/state/query executor,
+ // which has its own DSN and its own failure path. configured="false"
+ // (STATE_QUERY_DSN or its tenant password unset) always reports 1.
+ StateQueryReady = promauto.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "cyclops_cs_state_query_ready",
+ Help: "1 when the Kubernetes state query executor initialized at startup (or is intentionally disabled), 0 when a configured executor failed to come up.",
}, []string{"configured"})
UpstreamProxyDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{