Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cyclops): add namespace creation phase telemetry (#7737)
r33drichards6fb82f2798d7
diff --git a/libs/fleet/backend/handlers/namespaces.go b/libs/fleet/backend/handlers/namespaces.goindex 05a0060ed4..7fc66a1aeb 100644--- a/libs/fleet/backend/handlers/namespaces.go+++ b/libs/fleet/backend/handlers/namespaces.go@@ -6,6 +6,7 @@ import ("crypto/tls""crypto/x509""encoding/json"+ "errors""fmt""io""log/slog"@@ -17,6 +18,7 @@ import ("time""cyclops-cs-backend/identity"+ backendmetrics "cyclops-cs-backend/metrics""go.opentelemetry.io/otel/attribute""go.opentelemetry.io/otel/codes""go.opentelemetry.io/otel/trace"@@ -171,7 +173,7 @@ const (// cluster-wide to system:authenticated, so that probe succeeds instantly and// gates nothing.) Best-effort by design: failures are logged, never// surfaced — the namespace was created either way.-func (h Handlers) waitForNamespaceAdoption(ctx context.Context, name, userSub string) {+func (h Handlers) waitForNamespaceAdoption(ctx context.Context, name, userSub string) (string, error) {adoptionCtx, cancel := context.WithTimeout(ctx, adoptionWaitTimeout)defer cancel()for {@@ -182,18 +184,27 @@ func (h Handlers) waitForNamespaceAdoption(ctx context.Context, name, userSub st_, _ = io.Copy(io.Discard, resp.Body)resp.Body.Close()if resp.StatusCode >= 200 && resp.StatusCode < 300 {- return+ return "success", nil}}- if adoptionCtx.Err() != nil {- slog.Warn("namespace create: adoption wait timed out; returning anyway",- "namespace", name)- return+ if ctxErr := adoptionCtx.Err(); ctxErr != nil {+ result := "canceled"+ if ctxErr == context.DeadlineExceeded {+ result = "timeout"+ }+ slog.Warn("namespace create: adoption wait ended; returning anyway",+ "namespace", name, "result", result, "err", ctxErr)+ return result, errors.Join(ctxErr, err)}select {case <-time.After(adoptionWaitStep):case <-adoptionCtx.Done():- return+ ctxErr := adoptionCtx.Err()+ result := "canceled"+ if ctxErr == context.DeadlineExceeded {+ result = "timeout"+ }+ return result, errors.Join(ctxErr, err)}}}@@ -425,8 +436,14 @@ func (h Handlers) CreateNamespace(w http.ResponseWriter, r *http.Request) {return}- resp, err := h.k8sImpersonate(ctx, "POST", "/api/v1/namespaces", bytes.NewReader(body), user.ID)+ k8sStart := time.Now()+ k8sCtx, k8sSpan := handlerTracer().Start(ctx, "namespaces.create.k8s")+ resp, err := h.k8sImpersonate(k8sCtx, "POST", "/api/v1/namespaces", bytes.NewReader(body), user.ID)if err != nil {+ k8sSpan.RecordError(err)+ k8sSpan.SetStatus(codes.Error, "Kubernetes namespace create failed")+ k8sSpan.End()+ backendmetrics.RecordNamespaceCreatePhase("k8s_create", "error", time.Since(k8sStart))span.RecordError(err)span.SetStatus(codes.Error, "namespace create request failed")slog.Warn("namespace create: k8s request failed", "err", err)@@ -436,6 +453,17 @@ func (h Handlers) CreateNamespace(w http.ResponseWriter, r *http.Request) {defer resp.Body.Close()respBody, err := readBoundedK8sBody(resp.Body)+ k8sResult := fmt.Sprintf("http_%d", resp.StatusCode)+ k8sSpan.SetAttributes(attribute.Int("http.status_code", resp.StatusCode))+ if err != nil {+ k8sResult = "body_error"+ k8sSpan.RecordError(err)+ k8sSpan.SetStatus(codes.Error, "Kubernetes namespace response failed")+ } else if resp.StatusCode != http.StatusCreated {+ k8sSpan.SetStatus(codes.Error, "Kubernetes namespace create returned unexpected status")+ }+ k8sSpan.End()+ backendmetrics.RecordNamespaceCreatePhase("k8s_create", k8sResult, time.Since(k8sStart))if err != nil {span.RecordError(err)span.SetStatus(codes.Error, "namespace create response failed")@@ -466,12 +494,28 @@ func (h Handlers) CreateNamespace(w http.ResponseWriter, r *http.Request) {// it only succeeds once the RoleBindings exist — the same instant any other// in-namespace call becomes authorized. Best-effort: on timeout we still// return 201 (the namespace exists; adoption is merely late).- h.waitForNamespaceAdoption(ctx, req.Name, user.ID)+ adoptionStart := time.Now()+ adoptionCtx, adoptionSpan := handlerTracer().Start(ctx, "namespaces.create.adoption_wait")+ adoptionResult, adoptionErr := h.waitForNamespaceAdoption(adoptionCtx, req.Name, user.ID)+ if adoptionErr != nil {+ adoptionSpan.RecordError(adoptionErr)+ adoptionSpan.SetStatus(codes.Error, adoptionResult)+ }+ adoptionSpan.End()+ backendmetrics.RecordNamespaceCreatePhase("adoption_wait", adoptionResult, time.Since(adoptionStart))// Best-effort: provision the tenant's workload-OIDC credentials into the// new namespace so OSGym pool VMs here can mint a tenant-scoped OIDC// token (see provisionWorkloadOIDC; no-op when the feature is disabled).- h.provisionWorkloadOIDC(ctx, user.ID, req.Name)+ oidcStart := time.Now()+ oidcCtx, oidcSpan := handlerTracer().Start(ctx, "namespaces.create.workload_oidc")+ oidcResult, oidcErr := h.provisionWorkloadOIDC(oidcCtx, user.ID, req.Name)+ if oidcErr != nil {+ oidcSpan.RecordError(oidcErr)+ oidcSpan.SetStatus(codes.Error, oidcResult)+ }+ oidcSpan.End()+ backendmetrics.RecordNamespaceCreatePhase("workload_oidc", oidcResult, time.Since(oidcStart))// Parse the created namespace.var created struct {diff --git a/libs/fleet/backend/handlers/namespaces_test.go b/libs/fleet/backend/handlers/namespaces_test.goindex 0408b92ef2..d21231947a 100644--- a/libs/fleet/backend/handlers/namespaces_test.go+++ b/libs/fleet/backend/handlers/namespaces_test.go@@ -13,8 +13,10 @@ import ("cyclops-cs-backend/auth""go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/codes"sdktrace "go.opentelemetry.io/otel/sdk/trace""go.opentelemetry.io/otel/sdk/trace/tracetest"+ "github.com/prometheus/client_golang/prometheus")// withUser stamps a *auth.User on the request context the same way@@ -422,19 +424,80 @@ func TestCreateNamespace_CreatesTracingSpan(t *testing.T) {h.CreateNamespace(w, r)root.End()- var found bool+ spans := map[string]bool{}for _, span := range recorder.Ended() {- if span.Name() != "namespaces.create" {- continue+ spans[span.Name()] = true+ if span.Name() == "namespaces.create" {+ if got, want := span.Parent().SpanID(), root.SpanContext().SpanID(); got != want {+ t.Fatalf("parent span id = %s, want %s", got, want)+ }}- found = true- if got, want := span.Parent().SpanID(), root.SpanContext().SpanID(); got != want {- t.Fatalf("parent span id = %s, want %s", got, want)+ }+ for _, name := range []string{"namespaces.create", "namespaces.create.k8s", "namespaces.create.adoption_wait", "namespaces.create.workload_oidc"} {+ if !spans[name] {+ t.Errorf("expected %s span", name)}}- if !found {- t.Fatal("expected namespaces.create span")+}++func TestCreateNamespace_UnexpectedSuccessStatusMarksK8sSpanError(t *testing.T) {+ fk := newFakeK8s(http.StatusAccepted, `{}`)+ defer fk.server.Close()+ overrideK8sClient(fk.server.Client(), fk.server.URL, "fake-sa-token")++ recorder := tracetest.NewSpanRecorder()+ tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))+ previous := otel.GetTracerProvider()+ otel.SetTracerProvider(tp)+ t.Cleanup(func() { otel.SetTracerProvider(previous) })++ r := httptest.NewRequest(http.MethodPost, "/api/namespaces", strings.NewReader(`{"name":"my-workspace"}`))+ r = withUser(r, &auth.User{ID: "test-uuid"})+ w := httptest.NewRecorder()+ before := namespaceCreatePhaseHistogramCount(t, "k8s_create", "http_202")++ Handlers{}.CreateNamespace(w, r)++ if w.Code == http.StatusCreated {+ t.Fatalf("status = %d, want non-201 for unexpected Kubernetes status", w.Code)+ }+ after := namespaceCreatePhaseHistogramCount(t, "k8s_create", "http_202")+ if after != before+1 {+ t.Fatalf("http_202 metric count = %d, want %d", after, before+1)+ }++ for _, span := range recorder.Ended() {+ if span.Name() == "namespaces.create.k8s" {+ if span.Status().Code != codes.Error {+ t.Fatalf("k8s span status = %v, want error", span.Status().Code)+ }+ return+ }+ }+ t.Fatal("expected namespaces.create.k8s span")+}++func namespaceCreatePhaseHistogramCount(t *testing.T, phase, result string) uint64 {+ t.Helper()+ families, err := prometheus.DefaultGatherer.Gather()+ if err != nil {+ t.Fatal(err)+ }+ for _, family := range families {+ if family.GetName() != "cyclops_cs_namespace_create_phase_duration_seconds" {+ continue+ }+ for _, metric := range family.Metric {+ labels := map[string]string{}+ for _, pair := range metric.Label {+ labels[pair.GetName()] = pair.GetValue()+ }+ if labels["phase"] == phase && labels["result"] == result {+ return metric.GetHistogram().GetSampleCount()+ }+ }}+ return 0}func TestDeleteNamespace_Success(t *testing.T) {diff --git a/libs/fleet/backend/handlers/workload_oidc.go b/libs/fleet/backend/handlers/workload_oidc.goindex 35b02392f7..7fb0d6c853 100644--- a/libs/fleet/backend/handlers/workload_oidc.go+++ b/libs/fleet/backend/handlers/workload_oidc.go@@ -26,9 +26,9 @@ const workloadOIDCSecretName = "cua-workload-oidc"// (WorkloadAdmin nil). All failures are logged, never surfaced: the// namespace was created either way, and the OSGym Sandbox controller retries// until the Secret appears.-func (h Handlers) provisionWorkloadOIDC(ctx context.Context, userSub, namespace string) {+func (h Handlers) provisionWorkloadOIDC(ctx context.Context, userSub, namespace string) (string, error) {if h.WorkloadAdmin == nil {- return+ return "disabled", nil}tenant := identity.PersonalGroup(ctx, userSub)clientID, clientSecret, err := h.WorkloadAdmin.EnsureTenantWorkloadClient(@@ -37,17 +37,18 @@ func (h Handlers) provisionWorkloadOIDC(ctx context.Context, userSub, namespaceif err != nil {slog.Warn("workload-oidc: ensure tenant client failed","err", err, "namespace", namespace, "tenant", tenant)- return+ return "keycloak_error", err}if err := h.writeWorkloadOIDCSecret(ctx, userSub, namespace, clientID, clientSecret, tenant,); err != nil {slog.Warn("workload-oidc: write credentials Secret failed","err", err, "namespace", namespace)- return+ return "secret_error", err}slog.Info("workload-oidc: provisioned tenant credentials","namespace", namespace, "tenant", tenant, "client_id", clientID)+ return "success", nil}// writeWorkloadOIDCSecret creates the credentials Secret in the namespace viadiff --git a/libs/fleet/backend/handlers/workload_oidc_test.go b/libs/fleet/backend/handlers/workload_oidc_test.goindex c3503f65e7..042f1070e2 100644--- a/libs/fleet/backend/handlers/workload_oidc_test.go+++ b/libs/fleet/backend/handlers/workload_oidc_test.go@@ -15,7 +15,10 @@ func TestProvisionWorkloadOIDC_DisabledIsNoop(t *testing.T) {overrideK8sClient(fk.server.Client(), fk.server.URL, "fake-sa-token")h := Handlers{} // WorkloadAdmin == nil- h.provisionWorkloadOIDC(context.Background(), "sub-1", "mypool")+ result, err := h.provisionWorkloadOIDC(context.Background(), "sub-1", "mypool")+ if err != nil || result != "disabled" {+ t.Fatalf("result=%q err=%v, want disabled nil", result, err)+ }if len(fk.requests) != 0 {t.Fatalf("disabled feature made %d k8s request(s), want 0", len(fk.requests))diff --git a/libs/fleet/backend/metrics/metrics.go b/libs/fleet/backend/metrics/metrics.goindex 8201d3a94b..17c80ee322 100644--- a/libs/fleet/backend/metrics/metrics.go+++ b/libs/fleet/backend/metrics/metrics.go@@ -9,6 +9,7 @@// cyclops_cs_upstream_proxy_requests_total - counter, labels: target, status_code// cyclops_cs_upstream_proxy_duration_seconds - histogram, labels: target, status_code// cyclops_cs_active_requests - gauge+// cyclops_cs_namespace_create_phase_duration_seconds - histogram, labels: phase, result//// The metrics server is started separately on METRICS_ADDR (default :9091)// so that the main HTTP server on :8080 stays free of /metrics traffic.@@ -47,6 +48,12 @@ var (Help: "Number of HTTP requests currently being processed by the cyclops-cs backend.",})+ NamespaceCreatePhaseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{+ Name: "cyclops_cs_namespace_create_phase_duration_seconds",+ Help: "Latency of bounded phases in POST /api/namespaces.",+ Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},+ }, []string{"phase", "result"})+// Keycloak admin API SLIsKeycloakRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{Name: "cyclops_cs_keycloak_requests_total",@@ -250,6 +257,11 @@ func normalizePath(p string) string {}}+// RecordNamespaceCreatePhase observes a bounded namespace creation phase and result.+func RecordNamespaceCreatePhase(phase, result string, duration time.Duration) {+ NamespaceCreatePhaseDuration.WithLabelValues(phase, result).Observe(duration.Seconds())+}+// RecordKeycloakRequest records a Keycloak admin API operation result.// operation: CreateKey, ListKeys, DeleteKey, AdminLogin// success: true on 2xx / no error, false otherwisediff --git a/libs/fleet/backend/metrics/metrics_test.go b/libs/fleet/backend/metrics/metrics_test.goindex 25a3f3caa5..895c96b481 100644--- a/libs/fleet/backend/metrics/metrics_test.go+++ b/libs/fleet/backend/metrics/metrics_test.go@@ -4,6 +4,7 @@ import ("net/http""net/http/httptest""testing"+ "time""github.com/prometheus/client_golang/prometheus")@@ -128,3 +129,35 @@ func TestNormalizePath(t *testing.T) {}}}++func TestRecordNamespaceCreatePhase(t *testing.T) {+ before := namespaceCreatePhaseHistogramCount(t, "k8s_create", "http_201")+ RecordNamespaceCreatePhase("k8s_create", "http_201", 750*time.Millisecond)+ after := namespaceCreatePhaseHistogramCount(t, "k8s_create", "http_201")+ if after != before+1 {+ t.Fatalf("namespace create phase histogram count = %d, want %d", after, before+1)+ }+}++func namespaceCreatePhaseHistogramCount(t *testing.T, phase, result string) uint64 {+ t.Helper()+ families, err := prometheus.DefaultGatherer.Gather()+ if err != nil {+ t.Fatal(err)+ }+ for _, family := range families {+ if family.GetName() != "cyclops_cs_namespace_create_phase_duration_seconds" {+ continue+ }+ for _, metric := range family.Metric {+ labels := map[string]string{}+ for _, pair := range metric.Label {+ labels[pair.GetName()] = pair.GetValue()+ }+ if labels["phase"] == phase && labels["result"] == result {+ return metric.GetHistogram().GetSampleCount()+ }+ }+ }+ return 0+}