Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cyclops): add namespace creation phase telemetry (#7737)

r33drichards6fb82f2798d7
diff --git a/libs/fleet/backend/handlers/namespaces.go b/libs/fleet/backend/handlers/namespaces.go
index 05a0060ed4..7fc66a1aeb 100644
--- a/libs/fleet/backend/handlers/namespaces.go
+++ b/libs/fleet/backend/handlers/namespaces.go
@@ -6,6 +6,7 @@ import (
"crypto/tls"
"crypto/x509"
"encoding/json"
+ "errors"
"fmt"
"io"
"log/slog"
@@ -17,6 +18,7 @@ import (
"time"
"cyclops-cs-backend/identity"
+ backendmetrics "cyclops-cs-backend/metrics"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
@@ -171,7 +173,7 @@ const (
// cluster-wide to system:authenticated, so that probe succeeds instantly and
// gates nothing.) Best-effort by design: failures are logged, never
// surfaced — the namespace was created either way.
-func (h Handlers) waitForNamespaceAdoption(ctx context.Context, name, userSub string) {
+func (h Handlers) waitForNamespaceAdoption(ctx context.Context, name, userSub string) (string, error) {
adoptionCtx, cancel := context.WithTimeout(ctx, adoptionWaitTimeout)
defer cancel()
for {
@@ -182,18 +184,27 @@ func (h Handlers) waitForNamespaceAdoption(ctx context.Context, name, userSub st
_, _ = io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
- return
+ return "success", nil
}
}
- if adoptionCtx.Err() != nil {
- slog.Warn("namespace create: adoption wait timed out; returning anyway",
- "namespace", name)
- return
+ if ctxErr := adoptionCtx.Err(); ctxErr != nil {
+ result := "canceled"
+ if ctxErr == context.DeadlineExceeded {
+ result = "timeout"
+ }
+ slog.Warn("namespace create: adoption wait ended; returning anyway",
+ "namespace", name, "result", result, "err", ctxErr)
+ return result, errors.Join(ctxErr, err)
}
select {
case <-time.After(adoptionWaitStep):
case <-adoptionCtx.Done():
- return
+ ctxErr := adoptionCtx.Err()
+ result := "canceled"
+ if ctxErr == context.DeadlineExceeded {
+ result = "timeout"
+ }
+ return result, errors.Join(ctxErr, err)
}
}
}
@@ -425,8 +436,14 @@ func (h Handlers) CreateNamespace(w http.ResponseWriter, r *http.Request) {
return
}
- resp, err := h.k8sImpersonate(ctx, "POST", "/api/v1/namespaces", bytes.NewReader(body), user.ID)
+ k8sStart := time.Now()
+ k8sCtx, k8sSpan := handlerTracer().Start(ctx, "namespaces.create.k8s")
+ resp, err := h.k8sImpersonate(k8sCtx, "POST", "/api/v1/namespaces", bytes.NewReader(body), user.ID)
if err != nil {
+ k8sSpan.RecordError(err)
+ k8sSpan.SetStatus(codes.Error, "Kubernetes namespace create failed")
+ k8sSpan.End()
+ backendmetrics.RecordNamespaceCreatePhase("k8s_create", "error", time.Since(k8sStart))
span.RecordError(err)
span.SetStatus(codes.Error, "namespace create request failed")
slog.Warn("namespace create: k8s request failed", "err", err)
@@ -436,6 +453,17 @@ func (h Handlers) CreateNamespace(w http.ResponseWriter, r *http.Request) {
defer resp.Body.Close()
respBody, err := readBoundedK8sBody(resp.Body)
+ k8sResult := fmt.Sprintf("http_%d", resp.StatusCode)
+ k8sSpan.SetAttributes(attribute.Int("http.status_code", resp.StatusCode))
+ if err != nil {
+ k8sResult = "body_error"
+ k8sSpan.RecordError(err)
+ k8sSpan.SetStatus(codes.Error, "Kubernetes namespace response failed")
+ } else if resp.StatusCode != http.StatusCreated {
+ k8sSpan.SetStatus(codes.Error, "Kubernetes namespace create returned unexpected status")
+ }
+ k8sSpan.End()
+ backendmetrics.RecordNamespaceCreatePhase("k8s_create", k8sResult, time.Since(k8sStart))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "namespace create response failed")
@@ -466,12 +494,28 @@ func (h Handlers) CreateNamespace(w http.ResponseWriter, r *http.Request) {
// it only succeeds once the RoleBindings exist — the same instant any other
// in-namespace call becomes authorized. Best-effort: on timeout we still
// return 201 (the namespace exists; adoption is merely late).
- h.waitForNamespaceAdoption(ctx, req.Name, user.ID)
+ adoptionStart := time.Now()
+ adoptionCtx, adoptionSpan := handlerTracer().Start(ctx, "namespaces.create.adoption_wait")
+ adoptionResult, adoptionErr := h.waitForNamespaceAdoption(adoptionCtx, req.Name, user.ID)
+ if adoptionErr != nil {
+ adoptionSpan.RecordError(adoptionErr)
+ adoptionSpan.SetStatus(codes.Error, adoptionResult)
+ }
+ adoptionSpan.End()
+ backendmetrics.RecordNamespaceCreatePhase("adoption_wait", adoptionResult, time.Since(adoptionStart))
// Best-effort: provision the tenant's workload-OIDC credentials into the
// new namespace so OSGym pool VMs here can mint a tenant-scoped OIDC
// token (see provisionWorkloadOIDC; no-op when the feature is disabled).
- h.provisionWorkloadOIDC(ctx, user.ID, req.Name)
+ oidcStart := time.Now()
+ oidcCtx, oidcSpan := handlerTracer().Start(ctx, "namespaces.create.workload_oidc")
+ oidcResult, oidcErr := h.provisionWorkloadOIDC(oidcCtx, user.ID, req.Name)
+ if oidcErr != nil {
+ oidcSpan.RecordError(oidcErr)
+ oidcSpan.SetStatus(codes.Error, oidcResult)
+ }
+ oidcSpan.End()
+ backendmetrics.RecordNamespaceCreatePhase("workload_oidc", oidcResult, time.Since(oidcStart))
// Parse the created namespace.
var created struct {
diff --git a/libs/fleet/backend/handlers/namespaces_test.go b/libs/fleet/backend/handlers/namespaces_test.go
index 0408b92ef2..d21231947a 100644
--- a/libs/fleet/backend/handlers/namespaces_test.go
+++ b/libs/fleet/backend/handlers/namespaces_test.go
@@ -13,8 +13,10 @@ import (
"cyclops-cs-backend/auth"
"go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
+ "github.com/prometheus/client_golang/prometheus"
)
// withUser stamps a *auth.User on the request context the same way
@@ -422,19 +424,80 @@ func TestCreateNamespace_CreatesTracingSpan(t *testing.T) {
h.CreateNamespace(w, r)
root.End()
- var found bool
+ spans := map[string]bool{}
for _, span := range recorder.Ended() {
- if span.Name() != "namespaces.create" {
- continue
+ spans[span.Name()] = true
+ if span.Name() == "namespaces.create" {
+ if got, want := span.Parent().SpanID(), root.SpanContext().SpanID(); got != want {
+ t.Fatalf("parent span id = %s, want %s", got, want)
+ }
}
- found = true
- if got, want := span.Parent().SpanID(), root.SpanContext().SpanID(); got != want {
- t.Fatalf("parent span id = %s, want %s", got, want)
+ }
+ for _, name := range []string{"namespaces.create", "namespaces.create.k8s", "namespaces.create.adoption_wait", "namespaces.create.workload_oidc"} {
+ if !spans[name] {
+ t.Errorf("expected %s span", name)
}
}
- if !found {
- t.Fatal("expected namespaces.create span")
+}
+
+func TestCreateNamespace_UnexpectedSuccessStatusMarksK8sSpanError(t *testing.T) {
+ fk := newFakeK8s(http.StatusAccepted, `{}`)
+ defer fk.server.Close()
+ overrideK8sClient(fk.server.Client(), fk.server.URL, "fake-sa-token")
+
+ recorder := tracetest.NewSpanRecorder()
+ tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
+ previous := otel.GetTracerProvider()
+ otel.SetTracerProvider(tp)
+ t.Cleanup(func() { otel.SetTracerProvider(previous) })
+
+ r := httptest.NewRequest(http.MethodPost, "/api/namespaces", strings.NewReader(`{"name":"my-workspace"}`))
+ r = withUser(r, &auth.User{ID: "test-uuid"})
+ w := httptest.NewRecorder()
+ before := namespaceCreatePhaseHistogramCount(t, "k8s_create", "http_202")
+
+ Handlers{}.CreateNamespace(w, r)
+
+ if w.Code == http.StatusCreated {
+ t.Fatalf("status = %d, want non-201 for unexpected Kubernetes status", w.Code)
+ }
+ after := namespaceCreatePhaseHistogramCount(t, "k8s_create", "http_202")
+ if after != before+1 {
+ t.Fatalf("http_202 metric count = %d, want %d", after, before+1)
+ }
+
+ for _, span := range recorder.Ended() {
+ if span.Name() == "namespaces.create.k8s" {
+ if span.Status().Code != codes.Error {
+ t.Fatalf("k8s span status = %v, want error", span.Status().Code)
+ }
+ return
+ }
+ }
+ t.Fatal("expected namespaces.create.k8s span")
+}
+
+func namespaceCreatePhaseHistogramCount(t *testing.T, phase, result string) uint64 {
+ t.Helper()
+ families, err := prometheus.DefaultGatherer.Gather()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, family := range families {
+ if family.GetName() != "cyclops_cs_namespace_create_phase_duration_seconds" {
+ continue
+ }
+ for _, metric := range family.Metric {
+ labels := map[string]string{}
+ for _, pair := range metric.Label {
+ labels[pair.GetName()] = pair.GetValue()
+ }
+ if labels["phase"] == phase && labels["result"] == result {
+ return metric.GetHistogram().GetSampleCount()
+ }
+ }
}
+ return 0
}
func TestDeleteNamespace_Success(t *testing.T) {
diff --git a/libs/fleet/backend/handlers/workload_oidc.go b/libs/fleet/backend/handlers/workload_oidc.go
index 35b02392f7..7fb0d6c853 100644
--- a/libs/fleet/backend/handlers/workload_oidc.go
+++ b/libs/fleet/backend/handlers/workload_oidc.go
@@ -26,9 +26,9 @@ const workloadOIDCSecretName = "cua-workload-oidc"
// (WorkloadAdmin nil). All failures are logged, never surfaced: the
// namespace was created either way, and the OSGym Sandbox controller retries
// until the Secret appears.
-func (h Handlers) provisionWorkloadOIDC(ctx context.Context, userSub, namespace string) {
+func (h Handlers) provisionWorkloadOIDC(ctx context.Context, userSub, namespace string) (string, error) {
if h.WorkloadAdmin == nil {
- return
+ return "disabled", nil
}
tenant := identity.PersonalGroup(ctx, userSub)
clientID, clientSecret, err := h.WorkloadAdmin.EnsureTenantWorkloadClient(
@@ -37,17 +37,18 @@ func (h Handlers) provisionWorkloadOIDC(ctx context.Context, userSub, namespace
if err != nil {
slog.Warn("workload-oidc: ensure tenant client failed",
"err", err, "namespace", namespace, "tenant", tenant)
- return
+ return "keycloak_error", err
}
if err := h.writeWorkloadOIDCSecret(
ctx, userSub, namespace, clientID, clientSecret, tenant,
); err != nil {
slog.Warn("workload-oidc: write credentials Secret failed",
"err", err, "namespace", namespace)
- return
+ return "secret_error", err
}
slog.Info("workload-oidc: provisioned tenant credentials",
"namespace", namespace, "tenant", tenant, "client_id", clientID)
+ return "success", nil
}
// writeWorkloadOIDCSecret creates the credentials Secret in the namespace via
diff --git a/libs/fleet/backend/handlers/workload_oidc_test.go b/libs/fleet/backend/handlers/workload_oidc_test.go
index c3503f65e7..042f1070e2 100644
--- a/libs/fleet/backend/handlers/workload_oidc_test.go
+++ b/libs/fleet/backend/handlers/workload_oidc_test.go
@@ -15,7 +15,10 @@ func TestProvisionWorkloadOIDC_DisabledIsNoop(t *testing.T) {
overrideK8sClient(fk.server.Client(), fk.server.URL, "fake-sa-token")
h := Handlers{} // WorkloadAdmin == nil
- h.provisionWorkloadOIDC(context.Background(), "sub-1", "mypool")
+ result, err := h.provisionWorkloadOIDC(context.Background(), "sub-1", "mypool")
+ if err != nil || result != "disabled" {
+ t.Fatalf("result=%q err=%v, want disabled nil", result, err)
+ }
if len(fk.requests) != 0 {
t.Fatalf("disabled feature made %d k8s request(s), want 0", len(fk.requests))
diff --git a/libs/fleet/backend/metrics/metrics.go b/libs/fleet/backend/metrics/metrics.go
index 8201d3a94b..17c80ee322 100644
--- a/libs/fleet/backend/metrics/metrics.go
+++ b/libs/fleet/backend/metrics/metrics.go
@@ -9,6 +9,7 @@
// cyclops_cs_upstream_proxy_requests_total - counter, labels: target, status_code
// cyclops_cs_upstream_proxy_duration_seconds - histogram, labels: target, status_code
// cyclops_cs_active_requests - gauge
+// cyclops_cs_namespace_create_phase_duration_seconds - histogram, labels: phase, result
//
// The metrics server is started separately on METRICS_ADDR (default :9091)
// so that the main HTTP server on :8080 stays free of /metrics traffic.
@@ -47,6 +48,12 @@ var (
Help: "Number of HTTP requests currently being processed by the cyclops-cs backend.",
})
+ NamespaceCreatePhaseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+ Name: "cyclops_cs_namespace_create_phase_duration_seconds",
+ Help: "Latency of bounded phases in POST /api/namespaces.",
+ Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+ }, []string{"phase", "result"})
+
// Keycloak admin API SLIs
KeycloakRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "cyclops_cs_keycloak_requests_total",
@@ -250,6 +257,11 @@ func normalizePath(p string) string {
}
}
+// RecordNamespaceCreatePhase observes a bounded namespace creation phase and result.
+func RecordNamespaceCreatePhase(phase, result string, duration time.Duration) {
+ NamespaceCreatePhaseDuration.WithLabelValues(phase, result).Observe(duration.Seconds())
+}
+
// RecordKeycloakRequest records a Keycloak admin API operation result.
// operation: CreateKey, ListKeys, DeleteKey, AdminLogin
// success: true on 2xx / no error, false otherwise
diff --git a/libs/fleet/backend/metrics/metrics_test.go b/libs/fleet/backend/metrics/metrics_test.go
index 25a3f3caa5..895c96b481 100644
--- a/libs/fleet/backend/metrics/metrics_test.go
+++ b/libs/fleet/backend/metrics/metrics_test.go
@@ -4,6 +4,7 @@ import (
"net/http"
"net/http/httptest"
"testing"
+ "time"
"github.com/prometheus/client_golang/prometheus"
)
@@ -128,3 +129,35 @@ func TestNormalizePath(t *testing.T) {
}
}
}
+
+func TestRecordNamespaceCreatePhase(t *testing.T) {
+ before := namespaceCreatePhaseHistogramCount(t, "k8s_create", "http_201")
+ RecordNamespaceCreatePhase("k8s_create", "http_201", 750*time.Millisecond)
+ after := namespaceCreatePhaseHistogramCount(t, "k8s_create", "http_201")
+ if after != before+1 {
+ t.Fatalf("namespace create phase histogram count = %d, want %d", after, before+1)
+ }
+}
+
+func namespaceCreatePhaseHistogramCount(t *testing.T, phase, result string) uint64 {
+ t.Helper()
+ families, err := prometheus.DefaultGatherer.Gather()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, family := range families {
+ if family.GetName() != "cyclops_cs_namespace_create_phase_duration_seconds" {
+ continue
+ }
+ for _, metric := range family.Metric {
+ labels := map[string]string{}
+ for _, pair := range metric.Label {
+ labels[pair.GetName()] = pair.GetValue()
+ }
+ if labels["phase"] == phase && labels["result"] == result {
+ return metric.GetHistogram().GetSampleCount()
+ }
+ }
+ }
+ return 0
+}