Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(analytics): preserve bounded workload qualification diagnostics (#7545)

Francesco Bonacci95817401b8bf
diff --git a/libs/fleet/backend/handlers/attribution.go b/libs/fleet/backend/handlers/attribution.go
index 2f71d31c0c..dfcb597ae0 100644
--- a/libs/fleet/backend/handlers/attribution.go
+++ b/libs/fleet/backend/handlers/attribution.go
@@ -3,6 +3,7 @@ package handlers
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"net/http"
@@ -29,7 +30,10 @@ func (h Handlers) FleetAttributionQualification() productanalytics.SvcQualificat
reader := fleetAttributionFactsReader{handlers: h}
return func(ctx context.Context, r *http.Request, status int) productanalytics.SvcQualificationResult {
result := QualifySvcRequestResult(ctx, r, status, reader)
- return productanalytics.SvcQualificationResult{Qualifies: result.Qualifies, Reason: result.Reason}
+ return productanalytics.SvcQualificationResult{
+ Qualifies: result.Qualifies, Reason: result.Reason,
+ LookupStage: result.LookupStage, LookupErrorClass: result.LookupErrorClass,
+ }
}
}
@@ -46,16 +50,16 @@ func (reader fleetAttributionFactsReader) userSubject(ctx context.Context) (stri
func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.Context, namespace, sandbox string) (BoundClaim, error) {
subject, err := reader.userSubject(ctx)
if err != nil {
- return BoundClaim{}, err
+ return BoundClaim{}, errors.Join(err, attributionLookupError{stage: "sandbox", class: "missing_identity"})
}
sandboxPath := fmt.Sprintf("/apis/osgym.cua.ai/v1alpha1/namespaces/%s/osgymsandboxes/%s", url.PathEscape(namespace), url.PathEscape(sandbox))
response, err := reader.handlers.k8sImpersonate(ctx, http.MethodGet, sandboxPath, nil, subject)
if err != nil {
- return BoundClaim{}, err
+ return BoundClaim{}, errors.Join(err, classifyAttributionLookupError("sandbox", err, "request_failed"))
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
- return BoundClaim{}, fmt.Errorf("sandbox lookup returned HTTP %d", response.StatusCode)
+ return BoundClaim{}, attributionLookupStatusError("sandbox", response.StatusCode)
}
var sandboxPayload struct {
Metadata struct {
@@ -69,7 +73,7 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.C
} `json:"metadata"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&sandboxPayload); err != nil {
- return BoundClaim{}, err
+ return BoundClaim{}, errors.Join(err, classifyAttributionLookupError("sandbox", err, "invalid_response"))
}
var ownerName, ownerUID string
for _, owner := range sandboxPayload.Metadata.OwnerReferences {
@@ -77,7 +81,7 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.C
continue
}
if ownerName != "" {
- return BoundClaim{}, fmt.Errorf("sandbox has multiple claim controller owners")
+ return BoundClaim{}, attributionLookupError{stage: "sandbox", class: "invalid_binding"}
}
ownerName, ownerUID = owner.Name, owner.UID
}
@@ -88,11 +92,11 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.C
claimPath := fmt.Sprintf("/apis/osgym.cua.ai/v1alpha1/namespaces/%s/osgymsandboxclaims/%s", url.PathEscape(namespace), url.PathEscape(ownerName))
response, err = reader.handlers.k8sImpersonate(ctx, http.MethodGet, claimPath, nil, subject)
if err != nil {
- return BoundClaim{}, err
+ return BoundClaim{}, errors.Join(err, classifyAttributionLookupError("claim", err, "request_failed"))
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
- return BoundClaim{}, fmt.Errorf("claim lookup returned HTTP %d", response.StatusCode)
+ return BoundClaim{}, attributionLookupStatusError("claim", response.StatusCode)
}
var payload struct {
Metadata struct {
@@ -107,7 +111,7 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.C
} `json:"status"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil {
- return BoundClaim{}, err
+ return BoundClaim{}, errors.Join(err, classifyAttributionLookupError("claim", err, "invalid_response"))
}
return BoundClaim{
Claim: payload.Metadata.Name,
@@ -120,24 +124,24 @@ func (reader fleetAttributionFactsReader) ReadBoundClaimForSandbox(ctx context.C
func (reader fleetAttributionFactsReader) PoolExists(ctx context.Context, namespace, pool string) (bool, error) {
subject, err := reader.userSubject(ctx)
if err != nil {
- return false, err
+ return false, errors.Join(err, attributionLookupError{stage: "pool", class: "missing_identity"})
}
if pool == "" {
- return false, fmt.Errorf("missing pool")
+ return false, attributionLookupError{stage: "pool", class: "invalid_request"}
}
// Fleet SDK pools are native OSGymSandboxWarmPool resources. Keep the
// qualification lookup aligned with the resource the SDK actually creates.
path := fmt.Sprintf("/apis/osgym.cua.ai/v1alpha1/namespaces/%s/osgymsandboxwarmpools/%s", url.PathEscape(namespace), url.PathEscape(pool))
response, err := reader.handlers.k8sImpersonate(ctx, http.MethodGet, path, nil, subject)
if err != nil {
- return false, err
+ return false, errors.Join(err, classifyAttributionLookupError("pool", err, "request_failed"))
}
defer response.Body.Close()
if response.StatusCode == http.StatusNotFound {
return false, nil
}
if response.StatusCode != http.StatusOK {
- return false, fmt.Errorf("pool lookup returned HTTP %d", response.StatusCode)
+ return false, attributionLookupStatusError("pool", response.StatusCode)
}
return true, nil
}
@@ -147,8 +151,10 @@ func QualifySvcRequest(ctx context.Context, r *http.Request, status int, reader
}
type QualificationResult struct {
- Qualifies bool
- Reason string
+ Qualifies bool
+ Reason string
+ LookupStage string
+ LookupErrorClass string
}
func QualifySvcRequestResult(ctx context.Context, r *http.Request, status int, reader AttributionFactsReader) QualificationResult {
@@ -178,7 +184,7 @@ func QualifySvcRequestResult(ctx context.Context, r *http.Request, status int, r
}
bound, err := reader.ReadBoundClaimForSandbox(ctx, ns, sandbox)
if err != nil {
- return QualificationResult{Reason: "binding_lookup_failed"}
+ return attributionLookupRejection("binding_lookup_failed", "unknown", err)
}
if bound.Claim == "" {
return QualificationResult{Reason: "claim_missing"}
@@ -197,7 +203,7 @@ func QualifySvcRequestResult(ctx context.Context, r *http.Request, status int, r
}
pool, err := reader.PoolExists(ctx, ns, ns)
if err != nil {
- return QualificationResult{Reason: "pool_lookup_failed"}
+ return attributionLookupRejection("pool_lookup_failed", "pool", err)
}
if !pool {
return QualificationResult{Reason: "pool_missing"}
diff --git a/libs/fleet/backend/handlers/attribution_lookup.go b/libs/fleet/backend/handlers/attribution_lookup.go
new file mode 100644
index 0000000000..7e75830e5b
--- /dev/null
+++ b/libs/fleet/backend/handlers/attribution_lookup.go
@@ -0,0 +1,61 @@
+package handlers
+
+import (
+ "context"
+ "errors"
+ "net"
+ "net/http"
+)
+
+// attributionLookupError carries only bounded diagnostics. Never retain a raw
+// transport error, URL, response body, resource name, or identity in this value.
+// Readers join it with the original error to preserve the internal error chain;
+// qualification extracts only these fields and never renders the joined error.
+type attributionLookupError struct {
+ stage string
+ class string
+}
+
+func (err attributionLookupError) Error() string {
+ return "qualification lookup: " + err.stage + ": " + err.class
+}
+
+func classifyAttributionLookupError(stage string, err error, fallback string) attributionLookupError {
+ class := fallback
+ var networkError net.Error
+ switch {
+ case errors.Is(err, context.DeadlineExceeded):
+ class = "deadline_exceeded"
+ case errors.Is(err, context.Canceled):
+ class = "canceled"
+ case errors.As(err, &networkError) && networkError.Timeout():
+ class = "timeout"
+ }
+ return attributionLookupError{stage: stage, class: class}
+}
+
+func attributionLookupStatusError(stage string, status int) attributionLookupError {
+ class := "unexpected_status"
+ switch {
+ case status == http.StatusUnauthorized:
+ class = "unauthenticated"
+ case status == http.StatusForbidden:
+ class = "forbidden"
+ case status == http.StatusNotFound:
+ class = "not_found"
+ case status == http.StatusTooManyRequests:
+ class = "rate_limited"
+ case status >= 500 && status < 600:
+ class = "upstream_5xx"
+ }
+ return attributionLookupError{stage: stage, class: class}
+}
+
+func attributionLookupRejection(reason, fallbackStage string, err error) QualificationResult {
+ diagnostic := classifyAttributionLookupError(fallbackStage, err, "unknown")
+ var lookupError attributionLookupError
+ if errors.As(err, &lookupError) {
+ diagnostic = lookupError
+ }
+ return QualificationResult{Reason: reason, LookupStage: diagnostic.stage, LookupErrorClass: diagnostic.class}
+}
diff --git a/libs/fleet/backend/handlers/attribution_lookup_test.go b/libs/fleet/backend/handlers/attribution_lookup_test.go
new file mode 100644
index 0000000000..cf0cb7a58c
--- /dev/null
+++ b/libs/fleet/backend/handlers/attribution_lookup_test.go
@@ -0,0 +1,209 @@
+package handlers
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "cyclops-cs-backend/auth"
+ "cyclops-cs-backend/productanalytics"
+)
+
+type attributionRoundTripper func(*http.Request) (*http.Response, error)
+
+func (roundTrip attributionRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+ return roundTrip(r)
+}
+
+func TestQualificationLookupErrorClassesAreBounded(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ want string
+ }{
+ {"deadline", fmt.Errorf("private-resource: %w", context.DeadlineExceeded), "deadline_exceeded"},
+ {"cancel", &url.Error{URL: "https://private.example.test/token", Err: context.Canceled}, "canceled"},
+ {"network timeout", &net.DNSError{Err: "private-host", IsTimeout: true}, "timeout"},
+ {"request failure", errors.New("secret [email protected]"), "request_failed"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got := classifyAttributionLookupError("claim", tc.err, "request_failed")
+ if got.stage != "claim" || got.class != tc.want || got.Error() != "qualification lookup: claim: "+tc.want {
+ t.Fatalf("unexpected diagnostic: %#v", got)
+ }
+ })
+ }
+ result := attributionLookupRejection("binding_lookup_failed", "unknown", errors.New("private-resource"))
+ if result.Qualifies || result.LookupStage != "unknown" || result.LookupErrorClass != "unknown" {
+ t.Fatalf("untyped error must not invent a cause: %#v", result)
+ }
+}
+
+func TestQualificationLookupDiagnosticsPreserveFailClosedChecks(t *testing.T) {
+ for _, stage := range []string{"sandbox", "claim", "pool"} {
+ for _, tc := range []struct {
+ name string
+ status int
+ body string
+ err error
+ class string
+ }{
+ {name: "unauthenticated", status: 401, class: "unauthenticated"},
+ {name: "forbidden", status: 403, class: "forbidden"},
+ {name: "missing", status: 404, class: "not_found"},
+ {name: "throttled", status: 429, class: "rate_limited"},
+ {name: "unavailable", status: 503, class: "upstream_5xx"},
+ {name: "unexpected status", status: 400, class: "unexpected_status"},
+ {name: "invalid response", status: 200, body: `invalid [email protected]`, class: "invalid_response"},
+ {name: "deadline", err: context.DeadlineExceeded, class: "deadline_exceeded"},
+ {name: "transport", err: errors.New("private host token"), class: "request_failed"},
+ } {
+ if stage == "pool" && tc.name == "invalid response" {
+ continue // Pool existence checks status only; preserve that contract.
+ }
+ t.Run(stage+"/"+tc.name, func(t *testing.T) {
+ calls := 0
+ installAttributionTransport(t, attributionRoundTripper(func(r *http.Request) (*http.Response, error) {
+ calls++
+ if r.Header.Get("Impersonate-User") == "" || r.Header.Get("Impersonate-Group") == "" {
+ t.Error("lookup omitted namespace-scoping impersonation")
+ }
+ lookupStage, body := attributionLookupFixture(r.URL.Path)
+ status := http.StatusOK
+ if lookupStage == stage {
+ if err := tc.err; err != nil {
+ return nil, err
+ }
+ status, body = tc.status, tc.body
+ }
+ return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
+ }))
+ r := attributionDiagnosticRequest()
+ result := (Handlers{}).FleetAttributionQualification()(r.Context(), r, http.StatusOK)
+ wantReason, wantCalls := "binding_lookup_failed", 1
+ if stage == "claim" {
+ wantCalls = 2
+ } else if stage == "pool" {
+ wantReason, wantCalls = "pool_lookup_failed", 3
+ }
+ if stage == "pool" && tc.status == 404 {
+ if result.Qualifies || result.Reason != "pool_missing" || result.LookupStage != "" || result.LookupErrorClass != "" {
+ t.Fatalf("pool absence semantics changed: %#v", result)
+ }
+ return
+ }
+ if result.Qualifies || result.Reason != wantReason || result.LookupStage != stage || result.LookupErrorClass != tc.class || calls != wantCalls {
+ t.Fatalf("result=%#v calls=%d", result, calls)
+ }
+ if err := productanalytics.ValidateEvent(productanalytics.Event{Name: productanalytics.EventQualificationRejected, DistinctID: "synthetic-subject", Properties: map[string]any{
+ "reason": result.Reason, "qualification_lookup_stage": result.LookupStage, "qualification_error_class": result.LookupErrorClass,
+ }}); err != nil {
+ t.Fatalf("bounded diagnostic was rejected: %v", err)
+ }
+ })
+ }
+ }
+}
+
+// A successful workload response is not enough: reproduce a shared fact-read
+// deadline expiring at each stage without claiming this is the production cause.
+func TestQualificationSharedDeadlineIdentifiesFailingStage(t *testing.T) {
+ for _, stage := range []string{"sandbox", "claim", "pool"} {
+ t.Run(stage, func(t *testing.T) {
+ installAttributionTransport(t, attributionRoundTripper(func(r *http.Request) (*http.Response, error) {
+ lookupStage, body := attributionLookupFixture(r.URL.Path)
+ if lookupStage == stage {
+ <-r.Context().Done()
+ return nil, r.Context().Err()
+ }
+ return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
+ }))
+ r := attributionDiagnosticRequest()
+ ctx, cancel := context.WithTimeout(r.Context(), 250*time.Millisecond)
+ defer cancel()
+ result := (Handlers{}).FleetAttributionQualification()(ctx, r, http.StatusOK)
+ if result.Qualifies || result.LookupStage != stage || result.LookupErrorClass != "deadline_exceeded" {
+ t.Fatalf("result = %#v", result)
+ }
+ })
+ }
+}
+
+func TestAttributionReaderPreservesErrorChainButVerdictContainsOnlySafeFields(t *testing.T) {
+ for _, stage := range []string{"sandbox", "claim", "pool"} {
+ t.Run(stage, func(t *testing.T) {
+ cause := &net.DNSError{Name: "private-host.example.test", Err: "private transport detail", IsTimeout: true}
+ installAttributionTransport(t, attributionRoundTripper(func(r *http.Request) (*http.Response, error) {
+ lookupStage, body := attributionLookupFixture(r.URL.Path)
+ if lookupStage == stage {
+ return nil, cause
+ }
+ return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
+ }))
+ r := attributionDiagnosticRequest()
+ reader := fleetAttributionFactsReader{handlers: Handlers{}}
+ var err error
+ if stage == "pool" {
+ _, err = reader.PoolExists(r.Context(), "pool-1", "pool-1")
+ } else {
+ _, err = reader.ReadBoundClaimForSandbox(r.Context(), "pool-1", "sandbox-1")
+ }
+ var original *net.DNSError
+ var diagnostic attributionLookupError
+ if !errors.Is(err, cause) || !errors.As(err, &original) || original != cause || !errors.As(err, &diagnostic) {
+ t.Fatal("reader must preserve original error identity and typed diagnostic")
+ }
+ if diagnostic.stage != stage || diagnostic.class != "timeout" {
+ t.Fatalf("diagnostic = %#v", diagnostic)
+ }
+ result := QualifySvcRequestResult(r.Context(), r, http.StatusOK, reader)
+ wantReason := "binding_lookup_failed"
+ if stage == "pool" {
+ wantReason = "pool_lookup_failed"
+ }
+ if result != (QualificationResult{Reason: wantReason, LookupStage: stage, LookupErrorClass: "timeout"}) {
+ t.Fatalf("verdict must contain only bounded diagnostic fields: %#v", result)
+ }
+ })
+ }
+}
+
+func installAttributionTransport(t *testing.T, transport http.RoundTripper) {
+ t.Helper()
+ oldClient, oldServer, oldToken := k8sClient, k8sAPIServer, k8sSAToken
+ k8sClientOnce = sync.Once{}
+ overrideK8sClient(&http.Client{Transport: transport}, "http://kubernetes.example.test", "synthetic-token")
+ t.Cleanup(func() {
+ k8sClientOnce = sync.Once{}
+ k8sClient, k8sAPIServer, k8sSAToken = oldClient, oldServer, oldToken
+ })
+}
+
+func attributionDiagnosticRequest() *http.Request {
+ r := httptest.NewRequest(http.MethodPost, "/api/svc/pool-1/sandbox-1-server/tools", nil)
+ r.Pattern = "/api/svc/{namespace}/{service}/{path...}"
+ r.SetPathValue("namespace", "pool-1")
+ r.SetPathValue("service", "sandbox-1-server")
+ r.SetPathValue("path", "tools")
+ return r.WithContext(context.WithValue(r.Context(), auth.UserKey, &auth.User{ID: "synthetic-subject"}))
+}
+
+func attributionLookupFixture(path string) (string, string) {
+ switch {
+ case strings.Contains(path, "/osgymsandboxes/"):
+ return "sandbox", `{"metadata":{"ownerReferences":[{"apiVersion":"osgym.cua.ai/v1alpha1","kind":"OSGymSandboxClaim","name":"claim-1","uid":"claim-uid-1","controller":true}]}}`
+ case strings.Contains(path, "/osgymsandboxclaims/"):
+ return "claim", `{"metadata":{"name":"claim-1","uid":"claim-uid-1"},"status":{"phase":"Bound","sandbox":{"name":"sandbox-1"}}}`
+ default:
+ return "pool", `{}`
+ }
+}
diff --git a/libs/fleet/backend/productanalytics/events.go b/libs/fleet/backend/productanalytics/events.go
index 5ff2561ec8..7edcc7809a 100644
--- a/libs/fleet/backend/productanalytics/events.go
+++ b/libs/fleet/backend/productanalytics/events.go
@@ -112,6 +112,15 @@ var allowedProperties = map[string]struct{}{
"error_class": {}, "environment": {}, "instrumentation_version": {},
"identity_class": {},
"resource_type": {}, "reason": {},
+ "qualification_lookup_stage": {}, "qualification_error_class": {},
+}
+
+var allowedQualificationLookupStages = map[string]struct{}{"sandbox": {}, "claim": {}, "pool": {}, "unknown": {}}
+var allowedQualificationErrorClasses = map[string]struct{}{
+ "deadline_exceeded": {}, "canceled": {}, "timeout": {}, "unauthenticated": {}, "forbidden": {},
+ "not_found": {}, "rate_limited": {}, "upstream_5xx": {}, "unexpected_status": {},
+ "invalid_response": {}, "invalid_binding": {}, "missing_identity": {}, "invalid_request": {},
+ "request_failed": {}, "unknown": {},
}
var allowedResourceTypes = map[string]struct{}{"pool": {}, "template": {}, "claim": {}}
@@ -238,6 +247,23 @@ func ValidateEvent(event Event) error {
if event.Name == EventQualificationRejected && !hasReason {
return fmt.Errorf("qualification rejection event requires reason")
}
+ stage, hasStage := event.Properties["qualification_lookup_stage"]
+ class, hasClass := event.Properties["qualification_error_class"]
+ if hasStage || hasClass {
+ stageValue, stageOK := stage.(string)
+ classValue, classOK := class.(string)
+ _, stageAllowed := allowedQualificationLookupStages[stageValue]
+ _, classAllowed := allowedQualificationErrorClasses[classValue]
+ if !stageOK || !classOK || !stageAllowed || !classAllowed {
+ return fmt.Errorf("qualification lookup diagnostics require bounded stage and error class")
+ }
+ if event.Name != EventQualificationRejected || (reason != "binding_lookup_failed" && reason != "pool_lookup_failed") {
+ return fmt.Errorf("qualification lookup diagnostics require lookup rejection")
+ }
+ if (reason == "pool_lookup_failed") != (stageValue == "pool") {
+ return fmt.Errorf("qualification lookup stage must match rejection reason")
+ }
+ }
return nil
}
diff --git a/libs/fleet/backend/productanalytics/events_test.go b/libs/fleet/backend/productanalytics/events_test.go
index bec61e8d13..9650d1638f 100644
--- a/libs/fleet/backend/productanalytics/events_test.go
+++ b/libs/fleet/backend/productanalytics/events_test.go
@@ -187,6 +187,57 @@ func TestValidateEventRejectsUnsafeProperties(t *testing.T) {
}
}
+func TestQualificationLookupDiagnosticsRequireBoundedPairedValues(t *testing.T) {
+ valid := func() Event {
+ return Event{Name: EventQualificationRejected, DistinctID: "synthetic-subject", Properties: map[string]any{
+ "reason": "binding_lookup_failed", "qualification_lookup_stage": "claim", "qualification_error_class": "deadline_exceeded",
+ }}
+ }
+ for stage := range allowedQualificationLookupStages {
+ for class := range allowedQualificationErrorClasses {
+ event := valid()
+ event.Properties["qualification_lookup_stage"] = stage
+ event.Properties["qualification_error_class"] = class
+ if stage == "pool" {
+ event.Properties["reason"] = "pool_lookup_failed"
+ }
+ if err := ValidateEvent(event); err != nil {
+ t.Fatalf("bounded pair %s/%s rejected: %v", stage, class, err)
+ }
+ }
+ }
+ for _, tc := range []struct {
+ name string
+ mutate func(*Event)
+ }{
+ {"raw error", func(e *Event) { e.Properties["qualification_error_class"] = "https://private.example.test/secret" }},
+ {"raw stage", func(e *Event) { e.Properties["qualification_lookup_stage"] = "[email protected]" }},
+ {"non-string", func(e *Event) { e.Properties["qualification_error_class"] = 403 }},
+ {"missing class", func(e *Event) { delete(e.Properties, "qualification_error_class") }},
+ {"missing stage", func(e *Event) { delete(e.Properties, "qualification_lookup_stage") }},
+ {"empty class", func(e *Event) { e.Properties["qualification_error_class"] = "" }},
+ {"wrong event", func(e *Event) { e.Name = EventQualifyingWorkload }},
+ {"wrong reason", func(e *Event) { e.Properties["reason"] = "claim_mismatch" }},
+ {"pool stage binding reason", func(e *Event) { e.Properties["qualification_lookup_stage"] = "pool" }},
+ {"binding stage pool reason", func(e *Event) { e.Properties["reason"] = "pool_lookup_failed" }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ event := valid()
+ tc.mutate(&event)
+ if err := ValidateEvent(event); err == nil {
+ t.Fatal("invalid lookup diagnostic was accepted")
+ }
+ })
+ }
+ // Existing events without these optional diagnostics remain valid.
+ event := valid()
+ delete(event.Properties, "qualification_lookup_stage")
+ delete(event.Properties, "qualification_error_class")
+ if err := ValidateEvent(event); err != nil {
+ t.Fatalf("legacy rejection invalid: %v", err)
+ }
+}
+
func TestValidatePaymentFunnelEvents(t *testing.T) {
setupStart := Event{
Name: EventPaymentSetupStart, DistinctID: "subject-1",
diff --git a/libs/fleet/backend/productanalytics/middleware.go b/libs/fleet/backend/productanalytics/middleware.go
index 23a014de5a..6edfc9b2a2 100644
--- a/libs/fleet/backend/productanalytics/middleware.go
+++ b/libs/fleet/backend/productanalytics/middleware.go
@@ -18,8 +18,10 @@ import (
type SvcQualifier func(context.Context, *http.Request, int) bool
type SvcQualificationResult struct {
- Qualifies bool
- Reason string
+ Qualifies bool
+ Reason string
+ LookupStage string
+ LookupErrorClass string
}
type SvcQualification func(context.Context, *http.Request, int) SvcQualificationResult
@@ -147,31 +149,7 @@ func routeObserver(route string, capturer Capturer, spaClientID string, qualifie
ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 250*time.Millisecond)
defer cancel()
result := qualifier(ctx, r, status)
- if !result.Qualifies {
- if result.Reason != "" {
- capturer.Capture(Event{Name: EventQualificationRejected, DistinctID: user.ID, InsertID: traceID + ":qualification-rejected", Properties: map[string]any{
- "outcome": OutcomeFailure, "source": source, "principal_type": user.PrincipalType, "status_code": status,
- "error_class": errorClass(status), "identity_class": IdentityExternal, "reason": result.Reason,
- }})
- }
- return
- }
- properties := map[string]any{
- "outcome": OutcomeSuccess, "source": source, "principal_type": user.PrincipalType,
- "status_code": status, "error_class": "", "identity_class": IdentityExternal,
- }
- capturer.Capture(Event{
- Name: EventQualifyingWorkload, DistinctID: user.ID, InsertID: traceID,
- Properties: properties,
- })
- capturer.Capture(Event{
- // This marker is emitted for every qualifying request. Activation
- // queries must take each identity's earliest timestamp until the
- // backend has durable once-per-user state.
- Name: EventFleetActivation, DistinctID: user.ID,
- SetOnce: map[string]any{firstActivationProperty: time.Now().UTC().Format(time.RFC3339)},
- Properties: properties,
- })
+ captureSvcQualification(capturer, result, user, source, status, traceID)
}()
}
// Current pool creation finishes with the template request. Keep an
@@ -206,6 +184,42 @@ func routeObserver(route string, capturer Capturer, spaClientID string, qualifie
}
}
+// captureSvcQualification emits a verdict synchronously so all events from a
+// result can be validated together. The caller keeps fact reads off the request
+// path and invokes this only for external-classified users.
+func captureSvcQualification(capturer Capturer, result SvcQualificationResult, user *auth.User, source string, status int, traceID string) {
+ if !result.Qualifies {
+ if result.Reason != "" {
+ properties := map[string]any{
+ "outcome": OutcomeFailure, "source": source, "principal_type": user.PrincipalType, "status_code": status,
+ "error_class": errorClass(status), "identity_class": IdentityExternal, "reason": result.Reason,
+ }
+ if result.LookupStage != "" || result.LookupErrorClass != "" {
+ properties["qualification_lookup_stage"] = result.LookupStage
+ properties["qualification_error_class"] = result.LookupErrorClass
+ }
+ capturer.Capture(Event{Name: EventQualificationRejected, DistinctID: user.ID, InsertID: traceID + ":qualification-rejected", Properties: properties})
+ }
+ return
+ }
+ properties := map[string]any{
+ "outcome": OutcomeSuccess, "source": source, "principal_type": user.PrincipalType,
+ "status_code": status, "error_class": "", "identity_class": IdentityExternal,
+ }
+ capturer.Capture(Event{
+ Name: EventQualifyingWorkload, DistinctID: user.ID, InsertID: traceID,
+ Properties: properties,
+ })
+ capturer.Capture(Event{
+ // This marker is emitted for every qualifying request. Activation
+ // queries must take each identity's earliest timestamp until the
+ // backend has durable once-per-user state.
+ Name: EventFleetActivation, DistinctID: user.ID,
+ SetOnce: map[string]any{firstActivationProperty: time.Now().UTC().Format(time.RFC3339)},
+ Properties: properties,
+ })
+}
+
type activationTarget struct {
eventName string
suppressSuccess bool
diff --git a/libs/fleet/backend/productanalytics/middleware_test.go b/libs/fleet/backend/productanalytics/middleware_test.go
index 25fb255c9c..8c0a2b8680 100644
--- a/libs/fleet/backend/productanalytics/middleware_test.go
+++ b/libs/fleet/backend/productanalytics/middleware_test.go
@@ -4,10 +4,12 @@ import (
"bufio"
"bytes"
"context"
+ "encoding/json"
"net"
"net/http"
"net/http/httptest"
"strings"
+ "sync"
"testing"
"time"
@@ -386,23 +388,105 @@ func TestRouteObserverClassifiesPaymentRequiredWithoutChangingResponse(t *testin
}
func TestRouteObserverEmitsBoundedQualificationRejectionForExternalUsers(t *testing.T) {
- sink := make(eventChannel, 2)
- handler := RouteObserverWithQualification("/api/svc/{namespace}/{service}/{path...}", sink, "cyclops-cs-spa", func(context.Context, *http.Request, int) SvcQualificationResult {
- return SvcQualificationResult{Reason: "claim_mismatch"}
- })(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
- user := &auth.User{ID: "external-1", Email: "[email protected]", EmailVerified: true, AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}
- handler.ServeHTTP(httptest.NewRecorder(), observedRequest(http.MethodGet, "/api/svc/{namespace}/{service}/{path...}", "tools", user))
- var events []Event
- deadline := time.After(time.Second)
- for len(events) < 2 {
- select {
- case event := <-sink:
- events = append(events, event)
- case <-deadline:
- t.Fatalf("events = %#v", events)
+ for _, result := range []SvcQualificationResult{
+ {Reason: "claim_mismatch"},
+ {Reason: "binding_lookup_failed", LookupStage: "claim", LookupErrorClass: "deadline_exceeded"},
+ {Reason: "pool_lookup_failed", LookupStage: "pool", LookupErrorClass: "forbidden"},
+ } {
+ t.Run(result.Reason, func(t *testing.T) {
+ sink := make(eventChannel, 4)
+ handler := RouteObserverWithQualification("/api/svc/{namespace}/{service}/{path...}", sink, "cyclops-cs-spa", func(context.Context, *http.Request, int) SvcQualificationResult {
+ return result
+ })(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+ user := &auth.User{ID: "external-1", Email: "[email protected]", EmailVerified: true, AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}
+ handler.ServeHTTP(httptest.NewRecorder(), observedRequest(http.MethodGet, "/api/svc/{namespace}/{service}/{path...}", "tools", user))
+ events := map[string]Event{}
+ deadline := time.After(time.Second)
+ for i := 0; i < 2; i++ {
+ select {
+ case event := <-sink:
+ events[event.Name] = event
+ case <-deadline:
+ t.Fatalf("events = %#v", events)
+ }
+ }
+ rejected, ok := events[EventQualificationRejected]
+ if !ok || rejected.Properties["reason"] != result.Reason || rejected.DistinctID != user.ID || events[EventHTTPProxyRequest].Properties["outcome"] != OutcomeSuccess {
+ t.Fatalf("events = %#v", events)
+ }
+ stage, hasStage := rejected.Properties["qualification_lookup_stage"]
+ class, hasClass := rejected.Properties["qualification_error_class"]
+ if result.LookupStage == "" {
+ if hasStage || hasClass {
+ t.Fatal("non-lookup rejection acquired lookup diagnostics")
+ }
+ } else if stage != result.LookupStage || class != result.LookupErrorClass {
+ t.Fatalf("lookup diagnostics lost: %#v", rejected.Properties)
+ }
+ if err := ValidateEvent(rejected); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+func TestLookupDiagnosticDeliveryIsPrivateAndNeverEmitsActivation(t *testing.T) {
+ var mu sync.Mutex
+ var payloads []map[string]any
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var payload map[string]any
+ if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+ t.Error(err)
+ }
+ mu.Lock()
+ payloads = append(payloads, payload)
+ mu.Unlock()
+ w.WriteHeader(http.StatusOK)
+ }))
+ t.Cleanup(server.Close)
+ client := New(Config{
+ Enabled: true, Host: server.URL, ProjectToken: "phc_test", IdentityKey: "synthetic-key", Environment: "production",
+ QueueSize: 4, BatchSize: 1, FlushInterval: time.Hour, RequestTimeout: time.Second,
+ })
+ t.Cleanup(func() {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ if err := client.Shutdown(ctx); err != nil {
+ t.Error(err)
+ }
+ })
+ user := &auth.User{ID: "raw-synthetic-subject", Email: "[email protected]", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}
+ result := SvcQualificationResult{Reason: "pool_lookup_failed", LookupStage: "pool", LookupErrorClass: "deadline_exceeded"}
+ // Complete the synchronous producer, then flush before inspecting the whole
+ // collection. A test reading only the first events can miss extra activations.
+ captureSvcQualification(client, result, user, SourceSPA, http.StatusOK, "synthetic-trace")
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ if err := client.Shutdown(ctx); err != nil {
+ t.Fatal(err)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ seen := map[string]int{}
+ for _, payload := range payloads {
+ encoded, _ := json.Marshal(payload)
+ if bytes.Contains(encoded, []byte(user.ID)) || bytes.Contains(encoded, []byte(user.Email)) {
+ t.Fatal("analytics payload exposed raw identity")
+ }
+ event := payload["batch"].([]any)[0].(map[string]any)
+ name := event["event"].(string)
+ seen[name]++
+ if event["distinct_id"] != PseudonymForUserID(user.ID, "synthetic-key") {
+ t.Fatal("analytics identity was not pseudonymized")
+ }
+ if name == EventQualificationRejected {
+ properties := event["properties"].(map[string]any)
+ if properties["status_code"] != float64(200) || properties["error_class"] != "" || properties["qualification_lookup_stage"] != "pool" || properties["qualification_error_class"] != "deadline_exceeded" {
+ t.Fatalf("workload status or diagnostic changed: %#v", properties)
+ }
}
}
- if events[1].Name != EventQualificationRejected || events[1].Properties["reason"] != "claim_mismatch" || events[1].DistinctID != user.ID {
- t.Fatalf("event = %#v", events[1])
+ if seen[EventQualificationRejected] != 1 || len(seen) != 1 {
+ t.Fatalf("lookup failure must not emit workload success or activation: %#v", seen)
}
}
diff --git a/libs/fleet/docs/fleet-attribution-phase1.md b/libs/fleet/docs/fleet-attribution-phase1.md
index 3ec11ffdb3..1e9d0d3f21 100644
--- a/libs/fleet/docs/fleet-attribution-phase1.md
+++ b/libs/fleet/docs/fleet-attribution-phase1.md
@@ -8,10 +8,31 @@ Before Keycloak bootstrap, the SPA captures first-touch values from the current
The five stable fields are acquisition-channel neutral. They can represent email, SEO, partners, future channels, and organic or paid social traffic without provider-specific behavior. For example, both `utm_source=x&utm_medium=organic-social` and `utm_source=x&utm_medium=paid-social` are ordinary valid values. Provider SDKs, pixels, credentials, arbitrary provider payloads, and provider-specific success semantics are outside this contract. Pixels are optional downstream adapters and are not part of Phase 1.
+Use `content_id` for placement detail, not `utm_content` (an ignored unknown key). For the GitHub README CTA, use this capture-compatible URL:
+
+```text
+https://run.cua.ai/?utm_source=github&utm_medium=referral&utm_campaign=fleet_activation&content_id=repo_readme
+```
+
## Binding and qualification
-After Keycloak authentication, the SPA submits the validated record to the authenticated `/api/analytics/attribution` endpoint. The backend repeats the version, size, age, key, and value checks and accepts only the Fleet SPA principal. Stable authenticated identities emit `fleet_attribution_bound` unless verified `@trycua.com` evidence classifies them as internal. Missing or unverified email evidence is treated as external for growth measurement; unauthenticated identities or identities without a stable user ID remain unknown. PostHog receives the five fields only as set-once first-touch person properties under the keyed-HMAC pseudonym. The endpoint never accepts a URL, referrer, email, raw identity, arbitrary property, or provider payload, and attribution delivery cannot affect authentication or application readiness.
+After Keycloak authentication, the SPA submits the validated record to the authenticated `/api/analytics/attribution` endpoint. The backend repeats the version, size, age, key, and value checks and accepts only the Fleet SPA principal.
+
+For stable authenticated identities, verified `@trycua.com` email evidence or trusted admin-owner membership classifies the identity as internal. Without verified company-domain evidence, unresolved admin membership leaves the identity unknown. Unauthenticated identities and identities without a stable user ID also remain unknown. Resolved non-admin identities without verified company-domain evidence are external, including those with missing or unverified email evidence. Internal and unknown identities are discarded without emitting `fleet_attribution_bound`. External identities are eligible for that event, but configured subject exclusions suppress analytics delivery. PostHog receives the five fields only as set-once first-touch person properties under the keyed-HMAC pseudonym. The endpoint never accepts a URL, referrer, email, raw identity, arbitrary property, or provider payload, and attribution delivery cannot affect authentication or application readiness.
`QualifiesFleetAttribution` is a deterministic, side-effect-free predicate over explicit facts. Qualification requires authenticated namespace authorization, a persisted Bound claim whose sandbox identity matches the requested service, service exactly `<sandbox>-server`, an exact namespace pool, and a 2xx upstream response. Probe paths, redirects, upgrades, HEAD/OPTIONS, non-service routes, and Kubernetes/orchestration routes are excluded.
The backend derives the claim from Kubernetes state: the requested service identifies the sandbox, the sandbox's controller owner reference identifies the claim, and the persisted claim must point back to that sandbox. No SDK header or client-side claim inference is required. `qualifies` is a qualification verdict only; it is not an activation claim or emission.
+
+## Qualification lookup diagnostics
+
+Fact reads run after the workload response under one shared 250 ms deadline. A lookup failure remains a rejection: it does not emit a qualifying workload or activation, extend the deadline, or bypass ownership checks.
+
+For `fleet_workload_qualification_rejected`, `reason` retains `binding_lookup_failed` or `pool_lookup_failed`. Two optional properties distinguish fact-read failures:
+
+- `qualification_lookup_stage`: `sandbox`, `claim`, `pool`, or `unknown` when a reader cannot identify the binding stage.
+- `qualification_error_class`: `deadline_exceeded`, `canceled`, `timeout`, `unauthenticated`, `forbidden`, `not_found`, `rate_limited`, `upstream_5xx`, `unexpected_status`, `invalid_response`, `invalid_binding`, `missing_identity`, `invalid_request`, `request_failed`, or `unknown`.
+
+Both properties must be present together, use allowlisted values, and match a lookup-rejection reason. `request_failed` covers request construction, client availability, or transport errors that have no more specific classification. `unknown` does not imply a timeout. Pool HTTP 404 still produces `pool_missing`, without lookup-error diagnostics.
+
+`status_code` and `error_class` describe the workload's upstream response, not the Kubernetes fact read. An HTTP-200 request with rejected qualification is neither a proven failed workload nor a proven qualifying success. The diagnostic fields contain no raw errors, response bodies, URLs, resource names, email addresses, or user IDs. Existing pseudonymization and internal/unknown classification and exclusion rules still apply. Historical rejections without the new fields cannot establish the underlying failure cause.
diff --git a/libs/fleet/tests/fleet-attribution-doc.test.mjs b/libs/fleet/tests/fleet-attribution-doc.test.mjs
index 18c776997d..eb6d15ebcf 100644
--- a/libs/fleet/tests/fleet-attribution-doc.test.mjs
+++ b/libs/fleet/tests/fleet-attribution-doc.test.mjs
@@ -2,6 +2,16 @@ import assert from "node:assert/strict"
import { readFile } from "node:fs/promises"
import test from "node:test"
+test("lookup diagnostics distinguish qualification failures from workload failures", async () => {
+ const contract = await readFile(new URL("../docs/fleet-attribution-phase1.md", import.meta.url), "utf8")
+ assert.match(contract, /shared 250 ms deadline/i)
+ assert.match(contract, /qualification_lookup_stage/)
+ assert.match(contract, /qualification_error_class/)
+ assert.match(contract, /Both properties must be present together, use allowlisted values/i)
+ assert.match(contract, /neither a proven failed workload nor a proven qualifying success/i)
+ assert.match(contract, /Historical rejections without the new fields cannot establish the underlying failure cause/i)
+})
+
test("public contract is channel-neutral and says unknown query keys are ignored", async () => {
const contract = await readFile(new URL("../docs/fleet-attribution-phase1.md", import.meta.url), "utf8")
assert.match(contract, /acquisition-channel neutral/i)
@@ -13,3 +23,16 @@ test("public contract is channel-neutral and says unknown query keys are ignored
assert.match(contract, /internal and unknown identities are discarded/i)
assert.doesNotMatch(contract, /Unknown keys.*rejected/i)
})
+
+test("public contract matches backend identity classification and delivery exclusions", async () => {
+ const contract = await readFile(new URL("../docs/fleet-attribution-phase1.md", import.meta.url), "utf8")
+ assert.match(contract, /verified `@trycua\.com` email evidence or trusted admin-owner membership classifies the identity as internal/i)
+ assert.match(contract, /without verified company-domain evidence, unresolved admin membership leaves the identity unknown/i)
+ assert.match(contract, /unauthenticated identities and identities without a stable user ID also remain unknown/i)
+ assert.match(contract, /resolved non-admin identities without verified company-domain evidence are external, including those with missing or unverified email evidence/i)
+ assert.match(contract, /internal and unknown identities are discarded without emitting `fleet_attribution_bound`/i)
+ assert.match(contract, /configured subject exclusions suppress analytics delivery/i)
+ assert.match(contract, /only as set-once first-touch person properties under the keyed-HMAC pseudonym/i)
+ assert.match(contract, /never accepts a URL, referrer, email, raw identity, arbitrary property, or provider payload/i)
+ assert.doesNotMatch(contract, /missing or unverified email evidence is treated as external/i)
+})
diff --git a/libs/fleet/tests/fleet-attribution.test.ts b/libs/fleet/tests/fleet-attribution.test.ts
index 1335e36ad1..fcd5e65a2b 100644
--- a/libs/fleet/tests/fleet-attribution.test.ts
+++ b/libs/fleet/tests/fleet-attribution.test.ts
@@ -62,6 +62,52 @@ test("first touch survives a Keycloak round trip and only binds validated public
])
})
+test("GitHub README CTA preserves all four fields through a simulated Keycloak bind", async () => {
+ const cta = "https://run.cua.ai/?utm_source=github&utm_medium=referral&utm_campaign=fleet_activation&content_id=repo_readme"
+ const firstTouch = 1_700_000_000_000
+ const expected = {
+ version: 1,
+ capturedAt: firstTouch,
+ values: {
+ content_id: "repo_readme",
+ utm_source: "github",
+ utm_medium: "referral",
+ utm_campaign: "fleet_activation",
+ },
+ }
+
+ for (const href of [
+ cta,
+ `${cta}&utm_content=ignored&email=person%40example.test&identity=raw-subject&url=https%3A%2F%2Fexample.test`,
+ ]) {
+ const storage = memoryStorage()
+ captureFleetAttribution(href, { storage, now: () => firstTouch })
+ assert.deepEqual(JSON.parse(storage.getItem(ATTRIBUTION_STORAGE_KEY) ?? "null"), expected)
+
+ captureFleetAttribution(
+ "https://run.cua.ai/?code=oidc-code&state=opaque&utm_source=overwrite&content_id=overwrite",
+ { storage, now: () => firstTouch + 1_000 },
+ )
+ assert.deepEqual(JSON.parse(storage.getItem(ATTRIBUTION_STORAGE_KEY) ?? "null"), expected)
+
+ const bound: unknown[] = []
+ await bindStoredFleetAttribution({
+ bind: async (record, accessToken) => {
+ assert.equal(accessToken, "access-token")
+ bound.push(record)
+ },
+ }, {
+ storage,
+ now: () => firstTouch + 2_000,
+ getAccessToken: async () => "access-token",
+ })
+
+ // Exact records exclude the raw URL, email, identity, and unknown utm_content.
+ assert.deepEqual(bound, [expected])
+ assert.equal(storage.getItem(ATTRIBUTION_STORAGE_KEY), null)
+ }
+})
+
test("invalid, malformed, empty, oversized, and expired records never reach a binder", async () => {
const now = 1_700_000_000_000
const invalidStoredRecords = [