Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(analytics): preserve bounded workload qualification diagnostics (#7545)
Francesco Bonacci95817401b8bf
diff --git a/libs/fleet/backend/handlers/attribution.go b/libs/fleet/backend/handlers/attribution.goindex 2f71d31c0c..dfcb597ae0 100644--- a/libs/fleet/backend/handlers/attribution.go+++ b/libs/fleet/backend/handlers/attribution.go@@ -3,6 +3,7 @@ package handlersimport ("context""encoding/json"+ "errors""fmt""io""net/http"@@ -29,7 +30,10 @@ func (h Handlers) FleetAttributionQualification() productanalytics.SvcQualificatreader := fleetAttributionFactsReader{handlers: h}return func(ctx context.Context, r *http.Request, status int) productanalytics.SvcQualificationResult {result := QualifySvcRequestResult(ctx, r, status, reader)- return productanalytics.SvcQualificationResult{Qualifies: result.Qualifies, Reason: result.Reason}+ return productanalytics.SvcQualificationResult{+ Qualifies: result.Qualifies, Reason: result.Reason,+ LookupStage: result.LookupStage, LookupErrorClass: result.LookupErrorClass,+ }}}@@ -46,16 +50,16 @@ func (reader fleetAttributionFactsReader) userSubject(ctx context.Context) (strifunc (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.Context, namespace, sandbox string) (BoundClaim, error) {subject, err := reader.userSubject(ctx)if err != nil {- return BoundClaim{}, err+ return BoundClaim{}, errors.Join(err, attributionLookupError{stage: "sandbox", class: "missing_identity"})}sandboxPath := fmt.Sprintf("/apis/osgym.cua.ai/v1alpha1/namespaces/%s/osgymsandboxes/%s", url.PathEscape(namespace), url.PathEscape(sandbox))response, err := reader.handlers.k8sImpersonate(ctx, http.MethodGet, sandboxPath, nil, subject)if err != nil {- return BoundClaim{}, err+ return BoundClaim{}, errors.Join(err, classifyAttributionLookupError("sandbox", err, "request_failed"))}defer response.Body.Close()if response.StatusCode != http.StatusOK {- return BoundClaim{}, fmt.Errorf("sandbox lookup returned HTTP %d", response.StatusCode)+ return BoundClaim{}, attributionLookupStatusError("sandbox", response.StatusCode)}var sandboxPayload struct {Metadata struct {@@ -69,7 +73,7 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.C} `json:"metadata"`}if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&sandboxPayload); err != nil {- return BoundClaim{}, err+ return BoundClaim{}, errors.Join(err, classifyAttributionLookupError("sandbox", err, "invalid_response"))}var ownerName, ownerUID stringfor _, owner := range sandboxPayload.Metadata.OwnerReferences {@@ -77,7 +81,7 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.Ccontinue}if ownerName != "" {- return BoundClaim{}, fmt.Errorf("sandbox has multiple claim controller owners")+ return BoundClaim{}, attributionLookupError{stage: "sandbox", class: "invalid_binding"}}ownerName, ownerUID = owner.Name, owner.UID}@@ -88,11 +92,11 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.CclaimPath := fmt.Sprintf("/apis/osgym.cua.ai/v1alpha1/namespaces/%s/osgymsandboxclaims/%s", url.PathEscape(namespace), url.PathEscape(ownerName))response, err = reader.handlers.k8sImpersonate(ctx, http.MethodGet, claimPath, nil, subject)if err != nil {- return BoundClaim{}, err+ return BoundClaim{}, errors.Join(err, classifyAttributionLookupError("claim", err, "request_failed"))}defer response.Body.Close()if response.StatusCode != http.StatusOK {- return BoundClaim{}, fmt.Errorf("claim lookup returned HTTP %d", response.StatusCode)+ return BoundClaim{}, attributionLookupStatusError("claim", response.StatusCode)}var payload struct {Metadata struct {@@ -107,7 +111,7 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.C} `json:"status"`}if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil {- return BoundClaim{}, err+ return BoundClaim{}, errors.Join(err, classifyAttributionLookupError("claim", err, "invalid_response"))}return BoundClaim{Claim: payload.Metadata.Name,@@ -120,24 +124,24 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.Cfunc (reader fleetAttributionFactsReader) PoolExists(ctx context.Context, namespace, pool string) (bool, error) {subject, err := reader.userSubject(ctx)if err != nil {- return false, err+ return false, errors.Join(err, attributionLookupError{stage: "pool", class: "missing_identity"})}if pool == "" {- return false, fmt.Errorf("missing pool")+ return false, attributionLookupError{stage: "pool", class: "invalid_request"}}// Fleet SDK pools are native OSGymSandboxWarmPool resources. Keep the// qualification lookup aligned with the resource the SDK actually creates.path := fmt.Sprintf("/apis/osgym.cua.ai/v1alpha1/namespaces/%s/osgymsandboxwarmpools/%s", url.PathEscape(namespace), url.PathEscape(pool))response, err := reader.handlers.k8sImpersonate(ctx, http.MethodGet, path, nil, subject)if err != nil {- return false, err+ return false, errors.Join(err, classifyAttributionLookupError("pool", err, "request_failed"))}defer response.Body.Close()if response.StatusCode == http.StatusNotFound {return false, nil}if response.StatusCode != http.StatusOK {- return false, fmt.Errorf("pool lookup returned HTTP %d", response.StatusCode)+ return false, attributionLookupStatusError("pool", response.StatusCode)}return true, nil}@@ -147,8 +151,10 @@ func QualifySvcRequest(ctx context.Context, r *http.Request, status int, reader}type QualificationResult struct {- Qualifies bool- Reason string+ Qualifies bool+ Reason string+ LookupStage string+ LookupErrorClass string}func QualifySvcRequestResult(ctx context.Context, r *http.Request, status int, reader AttributionFactsReader) QualificationResult {@@ -178,7 +184,7 @@ func QualifySvcRequestResult(ctx context.Context, r *http.Request, status int, r}bound, err := reader.ReadBoundClaimForSandbox(ctx, ns, sandbox)if err != nil {- return QualificationResult{Reason: "binding_lookup_failed"}+ return attributionLookupRejection("binding_lookup_failed", "unknown", err)}if bound.Claim == "" {return QualificationResult{Reason: "claim_missing"}@@ -197,7 +203,7 @@ func QualifySvcRequestResult(ctx context.Context, r *http.Request, status int, r}pool, err := reader.PoolExists(ctx, ns, ns)if err != nil {- return QualificationResult{Reason: "pool_lookup_failed"}+ return attributionLookupRejection("pool_lookup_failed", "pool", err)}if !pool {return QualificationResult{Reason: "pool_missing"}diff --git a/libs/fleet/backend/handlers/attribution_lookup.go b/libs/fleet/backend/handlers/attribution_lookup.gonew file mode 100644index 0000000000..7e75830e5b--- /dev/null+++ b/libs/fleet/backend/handlers/attribution_lookup.go@@ -0,0 +1,61 @@+package handlers++import (+ "context"+ "errors"+ "net"+ "net/http"+)++// attributionLookupError carries only bounded diagnostics. Never retain a raw+// transport error, URL, response body, resource name, or identity in this value.+// Readers join it with the original error to preserve the internal error chain;+// qualification extracts only these fields and never renders the joined error.+type attributionLookupError struct {+ stage string+ class string+}++func (err attributionLookupError) Error() string {+ return "qualification lookup: " + err.stage + ": " + err.class+}++func classifyAttributionLookupError(stage string, err error, fallback string) attributionLookupError {+ class := fallback+ var networkError net.Error+ switch {+ case errors.Is(err, context.DeadlineExceeded):+ class = "deadline_exceeded"+ case errors.Is(err, context.Canceled):+ class = "canceled"+ case errors.As(err, &networkError) && networkError.Timeout():+ class = "timeout"+ }+ return attributionLookupError{stage: stage, class: class}+}++func attributionLookupStatusError(stage string, status int) attributionLookupError {+ class := "unexpected_status"+ switch {+ case status == http.StatusUnauthorized:+ class = "unauthenticated"+ case status == http.StatusForbidden:+ class = "forbidden"+ case status == http.StatusNotFound:+ class = "not_found"+ case status == http.StatusTooManyRequests:+ class = "rate_limited"+ case status >= 500 && status < 600:+ class = "upstream_5xx"+ }+ return attributionLookupError{stage: stage, class: class}+}++func attributionLookupRejection(reason, fallbackStage string, err error) QualificationResult {+ diagnostic := classifyAttributionLookupError(fallbackStage, err, "unknown")+ var lookupError attributionLookupError+ if errors.As(err, &lookupError) {+ diagnostic = lookupError+ }+ return QualificationResult{Reason: reason, LookupStage: diagnostic.stage, LookupErrorClass: diagnostic.class}+}diff --git a/libs/fleet/backend/handlers/attribution_lookup_test.go b/libs/fleet/backend/handlers/attribution_lookup_test.gonew file mode 100644index 0000000000..cf0cb7a58c--- /dev/null+++ b/libs/fleet/backend/handlers/attribution_lookup_test.go@@ -0,0 +1,209 @@+package handlers++import (+ "context"+ "errors"+ "fmt"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "sync"+ "testing"+ "time"++ "cyclops-cs-backend/auth"+ "cyclops-cs-backend/productanalytics"+)++type attributionRoundTripper func(*http.Request) (*http.Response, error)++func (roundTrip attributionRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {+ return roundTrip(r)+}++func TestQualificationLookupErrorClassesAreBounded(t *testing.T) {+ for _, tc := range []struct {+ name string+ err error+ want string+ }{+ {"deadline", fmt.Errorf("private-resource: %w", context.DeadlineExceeded), "deadline_exceeded"},+ {"cancel", &url.Error{URL: "https://private.example.test/token", Err: context.Canceled}, "canceled"},+ {"network timeout", &net.DNSError{Err: "private-host", IsTimeout: true}, "timeout"},+ {"request failure", errors.New("secret [email protected]"), "request_failed"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ got := classifyAttributionLookupError("claim", tc.err, "request_failed")+ if got.stage != "claim" || got.class != tc.want || got.Error() != "qualification lookup: claim: "+tc.want {+ t.Fatalf("unexpected diagnostic: %#v", got)+ }+ })+ }+ result := attributionLookupRejection("binding_lookup_failed", "unknown", errors.New("private-resource"))+ if result.Qualifies || result.LookupStage != "unknown" || result.LookupErrorClass != "unknown" {+ t.Fatalf("untyped error must not invent a cause: %#v", result)+ }+}++func TestQualificationLookupDiagnosticsPreserveFailClosedChecks(t *testing.T) {+ for _, stage := range []string{"sandbox", "claim", "pool"} {+ for _, tc := range []struct {+ name string+ status int+ body string+ err error+ class string+ }{+ {name: "unauthenticated", status: 401, class: "unauthenticated"},+ {name: "forbidden", status: 403, class: "forbidden"},+ {name: "missing", status: 404, class: "not_found"},+ {name: "throttled", status: 429, class: "rate_limited"},+ {name: "unavailable", status: 503, class: "upstream_5xx"},+ {name: "unexpected status", status: 400, class: "unexpected_status"},+ {name: "invalid response", status: 200, body: `invalid [email protected]`, class: "invalid_response"},+ {name: "deadline", err: context.DeadlineExceeded, class: "deadline_exceeded"},+ {name: "transport", err: errors.New("private host token"), class: "request_failed"},+ } {+ if stage == "pool" && tc.name == "invalid response" {+ continue // Pool existence checks status only; preserve that contract.+ }+ t.Run(stage+"/"+tc.name, func(t *testing.T) {+ calls := 0+ installAttributionTransport(t, attributionRoundTripper(func(r *http.Request) (*http.Response, error) {+ calls+++ if r.Header.Get("Impersonate-User") == "" || r.Header.Get("Impersonate-Group") == "" {+ t.Error("lookup omitted namespace-scoping impersonation")+ }+ lookupStage, body := attributionLookupFixture(r.URL.Path)+ status := http.StatusOK+ if lookupStage == stage {+ if err := tc.err; err != nil {+ return nil, err+ }+ status, body = tc.status, tc.body+ }+ return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil+ }))+ r := attributionDiagnosticRequest()+ result := (Handlers{}).FleetAttributionQualification()(r.Context(), r, http.StatusOK)+ wantReason, wantCalls := "binding_lookup_failed", 1+ if stage == "claim" {+ wantCalls = 2+ } else if stage == "pool" {+ wantReason, wantCalls = "pool_lookup_failed", 3+ }+ if stage == "pool" && tc.status == 404 {+ if result.Qualifies || result.Reason != "pool_missing" || result.LookupStage != "" || result.LookupErrorClass != "" {+ t.Fatalf("pool absence semantics changed: %#v", result)+ }+ return+ }+ if result.Qualifies || result.Reason != wantReason || result.LookupStage != stage || result.LookupErrorClass != tc.class || calls != wantCalls {+ t.Fatalf("result=%#v calls=%d", result, calls)+ }+ if err := productanalytics.ValidateEvent(productanalytics.Event{Name: productanalytics.EventQualificationRejected, DistinctID: "synthetic-subject", Properties: map[string]any{+ "reason": result.Reason, "qualification_lookup_stage": result.LookupStage, "qualification_error_class": result.LookupErrorClass,+ }}); err != nil {+ t.Fatalf("bounded diagnostic was rejected: %v", err)+ }+ })+ }+ }+}++// A successful workload response is not enough: reproduce a shared fact-read+// deadline expiring at each stage without claiming this is the production cause.+func TestQualificationSharedDeadlineIdentifiesFailingStage(t *testing.T) {+ for _, stage := range []string{"sandbox", "claim", "pool"} {+ t.Run(stage, func(t *testing.T) {+ installAttributionTransport(t, attributionRoundTripper(func(r *http.Request) (*http.Response, error) {+ lookupStage, body := attributionLookupFixture(r.URL.Path)+ if lookupStage == stage {+ <-r.Context().Done()+ return nil, r.Context().Err()+ }+ return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil+ }))+ r := attributionDiagnosticRequest()+ ctx, cancel := context.WithTimeout(r.Context(), 250*time.Millisecond)+ defer cancel()+ result := (Handlers{}).FleetAttributionQualification()(ctx, r, http.StatusOK)+ if result.Qualifies || result.LookupStage != stage || result.LookupErrorClass != "deadline_exceeded" {+ t.Fatalf("result = %#v", result)+ }+ })+ }+}++func TestAttributionReaderPreservesErrorChainButVerdictContainsOnlySafeFields(t *testing.T) {+ for _, stage := range []string{"sandbox", "claim", "pool"} {+ t.Run(stage, func(t *testing.T) {+ cause := &net.DNSError{Name: "private-host.example.test", Err: "private transport detail", IsTimeout: true}+ installAttributionTransport(t, attributionRoundTripper(func(r *http.Request) (*http.Response, error) {+ lookupStage, body := attributionLookupFixture(r.URL.Path)+ if lookupStage == stage {+ return nil, cause+ }+ return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil+ }))+ r := attributionDiagnosticRequest()+ reader := fleetAttributionFactsReader{handlers: Handlers{}}+ var err error+ if stage == "pool" {+ _, err = reader.PoolExists(r.Context(), "pool-1", "pool-1")+ } else {+ _, err = reader.ReadBoundClaimForSandbox(r.Context(), "pool-1", "sandbox-1")+ }+ var original *net.DNSError+ var diagnostic attributionLookupError+ if !errors.Is(err, cause) || !errors.As(err, &original) || original != cause || !errors.As(err, &diagnostic) {+ t.Fatal("reader must preserve original error identity and typed diagnostic")+ }+ if diagnostic.stage != stage || diagnostic.class != "timeout" {+ t.Fatalf("diagnostic = %#v", diagnostic)+ }+ result := QualifySvcRequestResult(r.Context(), r, http.StatusOK, reader)+ wantReason := "binding_lookup_failed"+ if stage == "pool" {+ wantReason = "pool_lookup_failed"+ }+ if result != (QualificationResult{Reason: wantReason, LookupStage: stage, LookupErrorClass: "timeout"}) {+ t.Fatalf("verdict must contain only bounded diagnostic fields: %#v", result)+ }+ })+ }+}++func installAttributionTransport(t *testing.T, transport http.RoundTripper) {+ t.Helper()+ oldClient, oldServer, oldToken := k8sClient, k8sAPIServer, k8sSAToken+ k8sClientOnce = sync.Once{}+ overrideK8sClient(&http.Client{Transport: transport}, "http://kubernetes.example.test", "synthetic-token")+ t.Cleanup(func() {+ k8sClientOnce = sync.Once{}+ k8sClient, k8sAPIServer, k8sSAToken = oldClient, oldServer, oldToken+ })+}++func attributionDiagnosticRequest() *http.Request {+ r := httptest.NewRequest(http.MethodPost, "/api/svc/pool-1/sandbox-1-server/tools", nil)+ r.Pattern = "/api/svc/{namespace}/{service}/{path...}"+ r.SetPathValue("namespace", "pool-1")+ r.SetPathValue("service", "sandbox-1-server")+ r.SetPathValue("path", "tools")+ return r.WithContext(context.WithValue(r.Context(), auth.UserKey, &auth.User{ID: "synthetic-subject"}))+}++func attributionLookupFixture(path string) (string, string) {+ switch {+ case strings.Contains(path, "/osgymsandboxes/"):+ return "sandbox", `{"metadata":{"ownerReferences":[{"apiVersion":"osgym.cua.ai/v1alpha1","kind":"OSGymSandboxClaim","name":"claim-1","uid":"claim-uid-1","controller":true}]}}`+ case strings.Contains(path, "/osgymsandboxclaims/"):+ return "claim", `{"metadata":{"name":"claim-1","uid":"claim-uid-1"},"status":{"phase":"Bound","sandbox":{"name":"sandbox-1"}}}`+ default:+ return "pool", `{}`+ }+}diff --git a/libs/fleet/backend/productanalytics/events.go b/libs/fleet/backend/productanalytics/events.goindex 5ff2561ec8..7edcc7809a 100644--- a/libs/fleet/backend/productanalytics/events.go+++ b/libs/fleet/backend/productanalytics/events.go@@ -112,6 +112,15 @@ var allowedProperties = map[string]struct{}{"error_class": {}, "environment": {}, "instrumentation_version": {},"identity_class": {},"resource_type": {}, "reason": {},+ "qualification_lookup_stage": {}, "qualification_error_class": {},+}++var allowedQualificationLookupStages = map[string]struct{}{"sandbox": {}, "claim": {}, "pool": {}, "unknown": {}}+var allowedQualificationErrorClasses = map[string]struct{}{+ "deadline_exceeded": {}, "canceled": {}, "timeout": {}, "unauthenticated": {}, "forbidden": {},+ "not_found": {}, "rate_limited": {}, "upstream_5xx": {}, "unexpected_status": {},+ "invalid_response": {}, "invalid_binding": {}, "missing_identity": {}, "invalid_request": {},+ "request_failed": {}, "unknown": {},}var allowedResourceTypes = map[string]struct{}{"pool": {}, "template": {}, "claim": {}}@@ -238,6 +247,23 @@ func ValidateEvent(event Event) error {if event.Name == EventQualificationRejected && !hasReason {return fmt.Errorf("qualification rejection event requires reason")}+ stage, hasStage := event.Properties["qualification_lookup_stage"]+ class, hasClass := event.Properties["qualification_error_class"]+ if hasStage || hasClass {+ stageValue, stageOK := stage.(string)+ classValue, classOK := class.(string)+ _, stageAllowed := allowedQualificationLookupStages[stageValue]+ _, classAllowed := allowedQualificationErrorClasses[classValue]+ if !stageOK || !classOK || !stageAllowed || !classAllowed {+ return fmt.Errorf("qualification lookup diagnostics require bounded stage and error class")+ }+ if event.Name != EventQualificationRejected || (reason != "binding_lookup_failed" && reason != "pool_lookup_failed") {+ return fmt.Errorf("qualification lookup diagnostics require lookup rejection")+ }+ if (reason == "pool_lookup_failed") != (stageValue == "pool") {+ return fmt.Errorf("qualification lookup stage must match rejection reason")+ }+ }return nil}diff --git a/libs/fleet/backend/productanalytics/events_test.go b/libs/fleet/backend/productanalytics/events_test.goindex bec61e8d13..9650d1638f 100644--- a/libs/fleet/backend/productanalytics/events_test.go+++ b/libs/fleet/backend/productanalytics/events_test.go@@ -187,6 +187,57 @@ func TestValidateEventRejectsUnsafeProperties(t *testing.T) {}}+func TestQualificationLookupDiagnosticsRequireBoundedPairedValues(t *testing.T) {+ valid := func() Event {+ return Event{Name: EventQualificationRejected, DistinctID: "synthetic-subject", Properties: map[string]any{+ "reason": "binding_lookup_failed", "qualification_lookup_stage": "claim", "qualification_error_class": "deadline_exceeded",+ }}+ }+ for stage := range allowedQualificationLookupStages {+ for class := range allowedQualificationErrorClasses {+ event := valid()+ event.Properties["qualification_lookup_stage"] = stage+ event.Properties["qualification_error_class"] = class+ if stage == "pool" {+ event.Properties["reason"] = "pool_lookup_failed"+ }+ if err := ValidateEvent(event); err != nil {+ t.Fatalf("bounded pair %s/%s rejected: %v", stage, class, err)+ }+ }+ }+ for _, tc := range []struct {+ name string+ mutate func(*Event)+ }{+ {"raw error", func(e *Event) { e.Properties["qualification_error_class"] = "https://private.example.test/secret" }},+ {"raw stage", func(e *Event) { e.Properties["qualification_lookup_stage"] = "[email protected]" }},+ {"non-string", func(e *Event) { e.Properties["qualification_error_class"] = 403 }},+ {"missing class", func(e *Event) { delete(e.Properties, "qualification_error_class") }},+ {"missing stage", func(e *Event) { delete(e.Properties, "qualification_lookup_stage") }},+ {"empty class", func(e *Event) { e.Properties["qualification_error_class"] = "" }},+ {"wrong event", func(e *Event) { e.Name = EventQualifyingWorkload }},+ {"wrong reason", func(e *Event) { e.Properties["reason"] = "claim_mismatch" }},+ {"pool stage binding reason", func(e *Event) { e.Properties["qualification_lookup_stage"] = "pool" }},+ {"binding stage pool reason", func(e *Event) { e.Properties["reason"] = "pool_lookup_failed" }},+ } {+ t.Run(tc.name, func(t *testing.T) {+ event := valid()+ tc.mutate(&event)+ if err := ValidateEvent(event); err == nil {+ t.Fatal("invalid lookup diagnostic was accepted")+ }+ })+ }+ // Existing events without these optional diagnostics remain valid.+ event := valid()+ delete(event.Properties, "qualification_lookup_stage")+ delete(event.Properties, "qualification_error_class")+ if err := ValidateEvent(event); err != nil {+ t.Fatalf("legacy rejection invalid: %v", err)+ }+}+func TestValidatePaymentFunnelEvents(t *testing.T) {setupStart := Event{Name: EventPaymentSetupStart, DistinctID: "subject-1",diff --git a/libs/fleet/backend/productanalytics/middleware.go b/libs/fleet/backend/productanalytics/middleware.goindex 23a014de5a..6edfc9b2a2 100644--- a/libs/fleet/backend/productanalytics/middleware.go+++ b/libs/fleet/backend/productanalytics/middleware.go@@ -18,8 +18,10 @@ import (type SvcQualifier func(context.Context, *http.Request, int) booltype SvcQualificationResult struct {- Qualifies bool- Reason string+ Qualifies bool+ Reason string+ LookupStage string+ LookupErrorClass string}type SvcQualification func(context.Context, *http.Request, int) SvcQualificationResult@@ -147,31 +149,7 @@ func routeObserver(route string, capturer Capturer, spaClientID string, qualifiectx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 250*time.Millisecond)defer cancel()result := qualifier(ctx, r, status)- if !result.Qualifies {- if result.Reason != "" {- capturer.Capture(Event{Name: EventQualificationRejected, DistinctID: user.ID, InsertID: traceID + ":qualification-rejected", Properties: map[string]any{- "outcome": OutcomeFailure, "source": source, "principal_type": user.PrincipalType, "status_code": status,- "error_class": errorClass(status), "identity_class": IdentityExternal, "reason": result.Reason,- }})- }- return- }- properties := map[string]any{- "outcome": OutcomeSuccess, "source": source, "principal_type": user.PrincipalType,- "status_code": status, "error_class": "", "identity_class": IdentityExternal,- }- capturer.Capture(Event{- Name: EventQualifyingWorkload, DistinctID: user.ID, InsertID: traceID,- Properties: properties,- })- capturer.Capture(Event{- // This marker is emitted for every qualifying request. Activation- // queries must take each identity's earliest timestamp until the- // backend has durable once-per-user state.- Name: EventFleetActivation, DistinctID: user.ID,- SetOnce: map[string]any{firstActivationProperty: time.Now().UTC().Format(time.RFC3339)},- Properties: properties,- })+ captureSvcQualification(capturer, result, user, source, status, traceID)}()}// Current pool creation finishes with the template request. Keep an@@ -206,6 +184,42 @@ func routeObserver(route string, capturer Capturer, spaClientID string, qualifie}}+// captureSvcQualification emits a verdict synchronously so all events from a+// result can be validated together. The caller keeps fact reads off the request+// path and invokes this only for external-classified users.+func captureSvcQualification(capturer Capturer, result SvcQualificationResult, user *auth.User, source string, status int, traceID string) {+ if !result.Qualifies {+ if result.Reason != "" {+ properties := map[string]any{+ "outcome": OutcomeFailure, "source": source, "principal_type": user.PrincipalType, "status_code": status,+ "error_class": errorClass(status), "identity_class": IdentityExternal, "reason": result.Reason,+ }+ if result.LookupStage != "" || result.LookupErrorClass != "" {+ properties["qualification_lookup_stage"] = result.LookupStage+ properties["qualification_error_class"] = result.LookupErrorClass+ }+ capturer.Capture(Event{Name: EventQualificationRejected, DistinctID: user.ID, InsertID: traceID + ":qualification-rejected", Properties: properties})+ }+ return+ }+ properties := map[string]any{+ "outcome": OutcomeSuccess, "source": source, "principal_type": user.PrincipalType,+ "status_code": status, "error_class": "", "identity_class": IdentityExternal,+ }+ capturer.Capture(Event{+ Name: EventQualifyingWorkload, DistinctID: user.ID, InsertID: traceID,+ Properties: properties,+ })+ capturer.Capture(Event{+ // This marker is emitted for every qualifying request. Activation+ // queries must take each identity's earliest timestamp until the+ // backend has durable once-per-user state.+ Name: EventFleetActivation, DistinctID: user.ID,+ SetOnce: map[string]any{firstActivationProperty: time.Now().UTC().Format(time.RFC3339)},+ Properties: properties,+ })+}+type activationTarget struct {eventName stringsuppressSuccess booldiff --git a/libs/fleet/backend/productanalytics/middleware_test.go b/libs/fleet/backend/productanalytics/middleware_test.goindex 25fb255c9c..8c0a2b8680 100644--- a/libs/fleet/backend/productanalytics/middleware_test.go+++ b/libs/fleet/backend/productanalytics/middleware_test.go@@ -4,10 +4,12 @@ import ("bufio""bytes""context"+ "encoding/json""net""net/http""net/http/httptest""strings"+ "sync""testing""time"@@ -386,23 +388,105 @@ func TestRouteObserverClassifiesPaymentRequiredWithoutChangingResponse(t *testin}func TestRouteObserverEmitsBoundedQualificationRejectionForExternalUsers(t *testing.T) {- sink := make(eventChannel, 2)- handler := RouteObserverWithQualification("/api/svc/{namespace}/{service}/{path...}", sink, "cyclops-cs-spa", func(context.Context, *http.Request, int) SvcQualificationResult {- return SvcQualificationResult{Reason: "claim_mismatch"}- })(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))- user := &auth.User{ID: "external-1", Email: "[email protected]", EmailVerified: true, AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}- handler.ServeHTTP(httptest.NewRecorder(), observedRequest(http.MethodGet, "/api/svc/{namespace}/{service}/{path...}", "tools", user))- var events []Event- deadline := time.After(time.Second)- for len(events) < 2 {- select {- case event := <-sink:- events = append(events, event)- case <-deadline:- t.Fatalf("events = %#v", events)+ for _, result := range []SvcQualificationResult{+ {Reason: "claim_mismatch"},+ {Reason: "binding_lookup_failed", LookupStage: "claim", LookupErrorClass: "deadline_exceeded"},+ {Reason: "pool_lookup_failed", LookupStage: "pool", LookupErrorClass: "forbidden"},+ } {+ t.Run(result.Reason, func(t *testing.T) {+ sink := make(eventChannel, 4)+ handler := RouteObserverWithQualification("/api/svc/{namespace}/{service}/{path...}", sink, "cyclops-cs-spa", func(context.Context, *http.Request, int) SvcQualificationResult {+ return result+ })(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))+ user := &auth.User{ID: "external-1", Email: "[email protected]", EmailVerified: true, AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}+ handler.ServeHTTP(httptest.NewRecorder(), observedRequest(http.MethodGet, "/api/svc/{namespace}/{service}/{path...}", "tools", user))+ events := map[string]Event{}+ deadline := time.After(time.Second)+ for i := 0; i < 2; i++ {+ select {+ case event := <-sink:+ events[event.Name] = event+ case <-deadline:+ t.Fatalf("events = %#v", events)+ }+ }+ rejected, ok := events[EventQualificationRejected]+ if !ok || rejected.Properties["reason"] != result.Reason || rejected.DistinctID != user.ID || events[EventHTTPProxyRequest].Properties["outcome"] != OutcomeSuccess {+ t.Fatalf("events = %#v", events)+ }+ stage, hasStage := rejected.Properties["qualification_lookup_stage"]+ class, hasClass := rejected.Properties["qualification_error_class"]+ if result.LookupStage == "" {+ if hasStage || hasClass {+ t.Fatal("non-lookup rejection acquired lookup diagnostics")+ }+ } else if stage != result.LookupStage || class != result.LookupErrorClass {+ t.Fatalf("lookup diagnostics lost: %#v", rejected.Properties)+ }+ if err := ValidateEvent(rejected); err != nil {+ t.Fatal(err)+ }+ })+ }+}++func TestLookupDiagnosticDeliveryIsPrivateAndNeverEmitsActivation(t *testing.T) {+ var mu sync.Mutex+ var payloads []map[string]any+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var payload map[string]any+ if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {+ t.Error(err)+ }+ mu.Lock()+ payloads = append(payloads, payload)+ mu.Unlock()+ w.WriteHeader(http.StatusOK)+ }))+ t.Cleanup(server.Close)+ client := New(Config{+ Enabled: true, Host: server.URL, ProjectToken: "phc_test", IdentityKey: "synthetic-key", Environment: "production",+ QueueSize: 4, BatchSize: 1, FlushInterval: time.Hour, RequestTimeout: time.Second,+ })+ t.Cleanup(func() {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ if err := client.Shutdown(ctx); err != nil {+ t.Error(err)+ }+ })+ user := &auth.User{ID: "raw-synthetic-subject", Email: "[email protected]", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}+ result := SvcQualificationResult{Reason: "pool_lookup_failed", LookupStage: "pool", LookupErrorClass: "deadline_exceeded"}+ // Complete the synchronous producer, then flush before inspecting the whole+ // collection. A test reading only the first events can miss extra activations.+ captureSvcQualification(client, result, user, SourceSPA, http.StatusOK, "synthetic-trace")+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()+ if err := client.Shutdown(ctx); err != nil {+ t.Fatal(err)+ }+ mu.Lock()+ defer mu.Unlock()+ seen := map[string]int{}+ for _, payload := range payloads {+ encoded, _ := json.Marshal(payload)+ if bytes.Contains(encoded, []byte(user.ID)) || bytes.Contains(encoded, []byte(user.Email)) {+ t.Fatal("analytics payload exposed raw identity")+ }+ event := payload["batch"].([]any)[0].(map[string]any)+ name := event["event"].(string)+ seen[name]+++ if event["distinct_id"] != PseudonymForUserID(user.ID, "synthetic-key") {+ t.Fatal("analytics identity was not pseudonymized")+ }+ if name == EventQualificationRejected {+ properties := event["properties"].(map[string]any)+ if properties["status_code"] != float64(200) || properties["error_class"] != "" || properties["qualification_lookup_stage"] != "pool" || properties["qualification_error_class"] != "deadline_exceeded" {+ t.Fatalf("workload status or diagnostic changed: %#v", properties)+ }}}- if events[1].Name != EventQualificationRejected || events[1].Properties["reason"] != "claim_mismatch" || events[1].DistinctID != user.ID {- t.Fatalf("event = %#v", events[1])+ if seen[EventQualificationRejected] != 1 || len(seen) != 1 {+ t.Fatalf("lookup failure must not emit workload success or activation: %#v", seen)}}diff --git a/libs/fleet/docs/fleet-attribution-phase1.md b/libs/fleet/docs/fleet-attribution-phase1.mdindex 3ec11ffdb3..1e9d0d3f21 100644--- a/libs/fleet/docs/fleet-attribution-phase1.md+++ b/libs/fleet/docs/fleet-attribution-phase1.md@@ -8,10 +8,31 @@ Before Keycloak bootstrap, the SPA captures first-touch values from the currentThe five stable fields are acquisition-channel neutral. They can represent email, SEO, partners, future channels, and organic or paid social traffic without provider-specific behavior. For example, both `utm_source=x&utm_medium=organic-social` and `utm_source=x&utm_medium=paid-social` are ordinary valid values. Provider SDKs, pixels, credentials, arbitrary provider payloads, and provider-specific success semantics are outside this contract. Pixels are optional downstream adapters and are not part of Phase 1.+Use `content_id` for placement detail, not `utm_content` (an ignored unknown key). For the GitHub README CTA, use this capture-compatible URL:++```text+https://run.cua.ai/?utm_source=github&utm_medium=referral&utm_campaign=fleet_activation&content_id=repo_readme+```+## Binding and qualification-After Keycloak authentication, the SPA submits the validated record to the authenticated `/api/analytics/attribution` endpoint. The backend repeats the version, size, age, key, and value checks and accepts only the Fleet SPA principal. Stable authenticated identities emit `fleet_attribution_bound` unless verified `@trycua.com` evidence classifies them as internal. Missing or unverified email evidence is treated as external for growth measurement; unauthenticated identities or identities without a stable user ID remain unknown. PostHog receives the five fields only as set-once first-touch person properties under the keyed-HMAC pseudonym. The endpoint never accepts a URL, referrer, email, raw identity, arbitrary property, or provider payload, and attribution delivery cannot affect authentication or application readiness.+After Keycloak authentication, the SPA submits the validated record to the authenticated `/api/analytics/attribution` endpoint. The backend repeats the version, size, age, key, and value checks and accepts only the Fleet SPA principal.++For stable authenticated identities, verified `@trycua.com` email evidence or trusted admin-owner membership classifies the identity as internal. Without verified company-domain evidence, unresolved admin membership leaves the identity unknown. Unauthenticated identities and identities without a stable user ID also remain unknown. Resolved non-admin identities without verified company-domain evidence are external, including those with missing or unverified email evidence. Internal and unknown identities are discarded without emitting `fleet_attribution_bound`. External identities are eligible for that event, but configured subject exclusions suppress analytics delivery. PostHog receives the five fields only as set-once first-touch person properties under the keyed-HMAC pseudonym. The endpoint never accepts a URL, referrer, email, raw identity, arbitrary property, or provider payload, and attribution delivery cannot affect authentication or application readiness.`QualifiesFleetAttribution` is a deterministic, side-effect-free predicate over explicit facts. Qualification requires authenticated namespace authorization, a persisted Bound claim whose sandbox identity matches the requested service, service exactly `<sandbox>-server`, an exact namespace pool, and a 2xx upstream response. Probe paths, redirects, upgrades, HEAD/OPTIONS, non-service routes, and Kubernetes/orchestration routes are excluded.The backend derives the claim from Kubernetes state: the requested service identifies the sandbox, the sandbox's controller owner reference identifies the claim, and the persisted claim must point back to that sandbox. No SDK header or client-side claim inference is required. `qualifies` is a qualification verdict only; it is not an activation claim or emission.++## Qualification lookup diagnostics++Fact reads run after the workload response under one shared 250 ms deadline. A lookup failure remains a rejection: it does not emit a qualifying workload or activation, extend the deadline, or bypass ownership checks.++For `fleet_workload_qualification_rejected`, `reason` retains `binding_lookup_failed` or `pool_lookup_failed`. Two optional properties distinguish fact-read failures:++- `qualification_lookup_stage`: `sandbox`, `claim`, `pool`, or `unknown` when a reader cannot identify the binding stage.+- `qualification_error_class`: `deadline_exceeded`, `canceled`, `timeout`, `unauthenticated`, `forbidden`, `not_found`, `rate_limited`, `upstream_5xx`, `unexpected_status`, `invalid_response`, `invalid_binding`, `missing_identity`, `invalid_request`, `request_failed`, or `unknown`.++Both properties must be present together, use allowlisted values, and match a lookup-rejection reason. `request_failed` covers request construction, client availability, or transport errors that have no more specific classification. `unknown` does not imply a timeout. Pool HTTP 404 still produces `pool_missing`, without lookup-error diagnostics.++`status_code` and `error_class` describe the workload's upstream response, not the Kubernetes fact read. An HTTP-200 request with rejected qualification is neither a proven failed workload nor a proven qualifying success. The diagnostic fields contain no raw errors, response bodies, URLs, resource names, email addresses, or user IDs. Existing pseudonymization and internal/unknown classification and exclusion rules still apply. Historical rejections without the new fields cannot establish the underlying failure cause.diff --git a/libs/fleet/tests/fleet-attribution-doc.test.mjs b/libs/fleet/tests/fleet-attribution-doc.test.mjsindex 18c776997d..eb6d15ebcf 100644--- a/libs/fleet/tests/fleet-attribution-doc.test.mjs+++ b/libs/fleet/tests/fleet-attribution-doc.test.mjs@@ -2,6 +2,16 @@ import assert from "node:assert/strict"import { readFile } from "node:fs/promises"import test from "node:test"+test("lookup diagnostics distinguish qualification failures from workload failures", async () => {+ const contract = await readFile(new URL("../docs/fleet-attribution-phase1.md", import.meta.url), "utf8")+ assert.match(contract, /shared 250 ms deadline/i)+ assert.match(contract, /qualification_lookup_stage/)+ assert.match(contract, /qualification_error_class/)+ assert.match(contract, /Both properties must be present together, use allowlisted values/i)+ assert.match(contract, /neither a proven failed workload nor a proven qualifying success/i)+ assert.match(contract, /Historical rejections without the new fields cannot establish the underlying failure cause/i)+})+test("public contract is channel-neutral and says unknown query keys are ignored", async () => {const contract = await readFile(new URL("../docs/fleet-attribution-phase1.md", import.meta.url), "utf8")assert.match(contract, /acquisition-channel neutral/i)@@ -13,3 +23,16 @@ test("public contract is channel-neutral and says unknown query keys are ignoredassert.match(contract, /internal and unknown identities are discarded/i)assert.doesNotMatch(contract, /Unknown keys.*rejected/i)})++test("public contract matches backend identity classification and delivery exclusions", async () => {+ const contract = await readFile(new URL("../docs/fleet-attribution-phase1.md", import.meta.url), "utf8")+ assert.match(contract, /verified `@trycua\.com` email evidence or trusted admin-owner membership classifies the identity as internal/i)+ assert.match(contract, /without verified company-domain evidence, unresolved admin membership leaves the identity unknown/i)+ assert.match(contract, /unauthenticated identities and identities without a stable user ID also remain unknown/i)+ assert.match(contract, /resolved non-admin identities without verified company-domain evidence are external, including those with missing or unverified email evidence/i)+ assert.match(contract, /internal and unknown identities are discarded without emitting `fleet_attribution_bound`/i)+ assert.match(contract, /configured subject exclusions suppress analytics delivery/i)+ assert.match(contract, /only as set-once first-touch person properties under the keyed-HMAC pseudonym/i)+ assert.match(contract, /never accepts a URL, referrer, email, raw identity, arbitrary property, or provider payload/i)+ assert.doesNotMatch(contract, /missing or unverified email evidence is treated as external/i)+})diff --git a/libs/fleet/tests/fleet-attribution.test.ts b/libs/fleet/tests/fleet-attribution.test.tsindex 1335e36ad1..fcd5e65a2b 100644--- a/libs/fleet/tests/fleet-attribution.test.ts+++ b/libs/fleet/tests/fleet-attribution.test.ts@@ -62,6 +62,52 @@ test("first touch survives a Keycloak round trip and only binds validated public])})+test("GitHub README CTA preserves all four fields through a simulated Keycloak bind", async () => {+ const cta = "https://run.cua.ai/?utm_source=github&utm_medium=referral&utm_campaign=fleet_activation&content_id=repo_readme"+ const firstTouch = 1_700_000_000_000+ const expected = {+ version: 1,+ capturedAt: firstTouch,+ values: {+ content_id: "repo_readme",+ utm_source: "github",+ utm_medium: "referral",+ utm_campaign: "fleet_activation",+ },+ }++ for (const href of [+ cta,+ `${cta}&utm_content=ignored&email=person%40example.test&identity=raw-subject&url=https%3A%2F%2Fexample.test`,+ ]) {+ const storage = memoryStorage()+ captureFleetAttribution(href, { storage, now: () => firstTouch })+ assert.deepEqual(JSON.parse(storage.getItem(ATTRIBUTION_STORAGE_KEY) ?? "null"), expected)++ captureFleetAttribution(+ "https://run.cua.ai/?code=oidc-code&state=opaque&utm_source=overwrite&content_id=overwrite",+ { storage, now: () => firstTouch + 1_000 },+ )+ assert.deepEqual(JSON.parse(storage.getItem(ATTRIBUTION_STORAGE_KEY) ?? "null"), expected)++ const bound: unknown[] = []+ await bindStoredFleetAttribution({+ bind: async (record, accessToken) => {+ assert.equal(accessToken, "access-token")+ bound.push(record)+ },+ }, {+ storage,+ now: () => firstTouch + 2_000,+ getAccessToken: async () => "access-token",+ })++ // Exact records exclude the raw URL, email, identity, and unknown utm_content.+ assert.deepEqual(bound, [expected])+ assert.equal(storage.getItem(ATTRIBUTION_STORAGE_KEY), null)+ }+})+test("invalid, malformed, empty, oversized, and expired records never reach a binder", async () => {const now = 1_700_000_000_000const invalidStoredRecords = [