Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cyclops): private API-only account lookup (#7459)
Francesco Bonacci032073fb941b
diff --git a/libs/fleet/backend/Dockerfile b/libs/fleet/backend/Dockerfileindex 28b726bdc9..8632509957 100644--- a/libs/fleet/backend/Dockerfile+++ b/libs/fleet/backend/Dockerfile@@ -11,6 +11,7 @@ RUN go install github.com/swaggo/swag/cmd/[email protected] && \/go/bin/swag init -g main.go -o ./docs --parseDependency --parseInternalRUN CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-cs-backend . \&& CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-db-migrate ./cmd/db-migrate \+ && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/account-lookup-backfill ./cmd/account-lookup-backfill \&& CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-usage-meter ./cmd/usage-meterFROM --platform=linux/amd64 rust:1.94-bookworm AS rust-builder@@ -24,6 +25,7 @@ FROM gcr.io/distroless/cc-debian12:nonroot AS runtimeWORKDIR /appCOPY --from=go-builder /out/cyclops-cs-backend /app/cyclops-cs-backendCOPY --from=go-builder /out/cyclops-db-migrate /app/cyclops-db-migrate+COPY --from=go-builder /out/account-lookup-backfill /app/account-lookup-backfillCOPY --from=go-builder /out/cyclops-usage-meter /app/cyclops-usage-meterCOPY --from=rust-builder /out-k8s-state-projector /app/k8s-state-projectorUSER nonroot:nonrootdiff --git a/libs/fleet/backend/accountlookup/service.go b/libs/fleet/backend/accountlookup/service.gonew file mode 100644index 0000000000..29079b9e22--- /dev/null+++ b/libs/fleet/backend/accountlookup/service.go@@ -0,0 +1,184 @@+// Package accountlookup resolves analytics identities only inside the admin boundary.+package accountlookup++import (+ "context"+ "errors"+ "net/http"+ "regexp"+ "sync"+ "time"++ "cyclops-cs-backend/auth"+ "cyclops-cs-backend/internal/redactederror"+ "cyclops-cs-backend/keycloak"+ "cyclops-cs-backend/productanalytics"+)++var (+ ErrUnavailable = errors.New("account lookup unavailable")+ ErrRateLimited = errors.New("account lookup rate limited")+ ErrInvalid = errors.New("invalid exact account lookup")+ pseudonymPattern = regexp.MustCompile(`^u_[0-9a-f]{64}$`)+ accountPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@-]{0,254}$`)+)++type Index interface {+ Record(context.Context, string, string, string, string) error+ Resolve(context.Context, string, string, string) (string, bool, error)+ Complete(context.Context, string, string) (bool, error)+ Audit(context.Context, string, string) error+ Allow(context.Context, string) (bool, error)+}++type Directory interface {+ LookupAccount(context.Context, string) (*keycloak.Account, error)+}+type Request struct {+ Kind string `json:"kind"`+ Value string `json:"value"`+}+type Account struct {+ keycloak.Account+ Pseudonym string `json:"pseudonym"`+ IdentityClass productanalytics.IdentityClass `json:"identity_class"`+ Excluded bool `json:"excluded"`+}+type Result struct {+ Status string `json:"status"`+ BackfillComplete bool `json:"backfill_complete"`+ Account *Account `json:"account,omitempty"`+}++type Service struct {+ index Index+ directory Directory+ realm, key, keyID string+ excluded map[string]bool+ queue chan string+ wg sync.WaitGroup+}++// KeyID scopes private index rows to the configured key without storing the key.+func KeyID(key string) string {+ return productanalytics.PseudonymForUserID("account-lookup-key-version-v1", key)+}++func New(ctx context.Context, index Index, directory Directory, realm, key string, excluded []string) *Service {+ s := &Service{index: index, directory: directory, realm: realm, key: key, keyID: KeyID(key), excluded: map[string]bool{}, queue: make(chan string, 256)}+ for _, id := range excluded {+ s.excluded[id] = true+ }+ s.wg.Add(1)+ go s.run(ctx)+ return s+}++func (s *Service) Wait() { s.wg.Wait() }++// Denials never disclose an account, even when their best-effort audit fails.+func (s *Service) AuditDenied(ctx context.Context, actor string) {+ if s != nil && s.index != nil && actor != "" {+ _ = s.index.Audit(ctx, actor, "forbidden")+ }+}++func (s *Service) Lookup(ctx context.Context, actor string, input Request) (Result, error) {+ if s == nil || s.index == nil || s.directory == nil || s.key == "" || s.realm == "" {+ return Result{}, ErrUnavailable+ }+ allowed, err := s.index.Allow(ctx, actor)+ if err != nil {+ return Result{}, redactederror.New(ErrUnavailable.Error(), errors.Join(ErrUnavailable, err))+ }+ if !allowed {+ return s.finish(ctx, actor, "rate_limited", Result{}, ErrRateLimited)+ }+ if !((input.Kind == "pseudonym" && pseudonymPattern.MatchString(input.Value)) || (input.Kind == "account_id" && accountPattern.MatchString(input.Value))) {+ return s.finish(ctx, actor, "invalid", Result{}, ErrInvalid)+ }+ complete, err := s.index.Complete(ctx, s.realm, s.keyID)+ if err != nil {+ return s.finish(ctx, actor, "unavailable", Result{}, redactederror.New(ErrUnavailable.Error(), errors.Join(ErrUnavailable, err)))+ }+ result := Result{BackfillComplete: complete}+ subject := input.Value+ if input.Kind == "pseudonym" {+ var found bool+ subject, found, err = s.index.Resolve(ctx, s.realm, s.keyID, input.Value)+ if err != nil {+ return s.finish(ctx, actor, "unavailable", Result{}, redactederror.New(ErrUnavailable.Error(), errors.Join(ErrUnavailable, err)))+ }+ if !found {+ result.Status = "mapping_missing"+ return s.finish(ctx, actor, result.Status, result, nil)+ }+ if productanalytics.PseudonymForUserID(subject, s.key) != input.Value {+ return s.finish(ctx, actor, "unavailable", Result{}, ErrUnavailable)+ }+ }+ account, err := s.directory.LookupAccount(ctx, subject)+ if err != nil {+ return s.finish(ctx, actor, "unavailable", Result{}, redactederror.New(ErrUnavailable.Error(), errors.Join(ErrUnavailable, err)))+ }+ if account == nil {+ result.Status = "account_missing"+ return s.finish(ctx, actor, result.Status, result, nil)+ }+ if account.ID != subject {+ return s.finish(ctx, actor, "unavailable", Result{}, ErrUnavailable)+ }+ result.Status = "found"+ result.Account = &Account{Account: *account, Pseudonym: productanalytics.PseudonymForUserID(subject, s.key), IdentityClass: productanalytics.ClassifyIdentity(&auth.User{ID: subject, Email: account.Email, EmailVerified: account.EmailVerified}), Excluded: s.excluded[subject]}+ return s.finish(ctx, actor, result.Status, result, nil)+}++// Do not disclose identity when the restricted audit cannot be committed.+func (s *Service) finish(ctx context.Context, actor, outcome string, result Result, err error) (Result, error) {+ if auditErr := s.index.Audit(ctx, actor, outcome); auditErr != nil {+ return Result{}, redactederror.New(ErrUnavailable.Error(), errors.Join(ErrUnavailable, err, auditErr))+ }+ return result, err+}++// Observe runs after token validation/owner normalization. The bounded queue+// never blocks authentication or workloads, including during a DB outage.+func (s *Service) Observe(next http.Handler) http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ u := auth.GetUser(r.Context())+ if s != nil && s.key != "" && u != nil && u.ID != "" && ((u.PrincipalType == auth.PrincipalTypeUser && (u.AZP == "cyclops-cs-spa" || u.AZP == "cua-cli" || u.AZP == "cua-desktop")) || u.PrincipalType == auth.PrincipalTypeUserKey) {+ select {+ case s.queue <- u.ID:+ default:+ }+ }+ next.ServeHTTP(w, r)+ })+}++func (s *Service) run(ctx context.Context) {+ defer s.wg.Done()+ seen := map[string]time.Time{}+ for {+ select {+ case <-ctx.Done():+ return+ case subject := <-s.queue:+ if time.Now().Before(seen[subject]) {+ continue+ }+ if s.index == nil || s.key == "" {+ continue+ }+ callCtx, cancel := context.WithTimeout(ctx, 2*time.Second)+ err := s.index.Record(callCtx, s.realm, s.keyID, productanalytics.PseudonymForUserID(subject, s.key), subject)+ cancel()+ if err == nil {+ if len(seen) >= 4096 {+ clear(seen)+ }+ seen[subject] = time.Now().Add(time.Hour)+ }+ }+ }+}diff --git a/libs/fleet/backend/accountlookup/service_test.go b/libs/fleet/backend/accountlookup/service_test.gonew file mode 100644index 0000000000..e11ecd7249--- /dev/null+++ b/libs/fleet/backend/accountlookup/service_test.go@@ -0,0 +1,213 @@+package accountlookup++import (+ "context"+ "errors"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "cyclops-cs-backend/auth"+ "cyclops-cs-backend/keycloak"+ "cyclops-cs-backend/productanalytics"+)++type fakeIndex struct {+ subject string+ complete, limited bool+ err, auditErr error+ outcomes []string+ recorded chan string+}++func (f *fakeIndex) Record(_ context.Context, _, _, pseudo, subject string) error {+ if f.recorded != nil {+ f.recorded <- pseudo + ":" + subject+ }+ return f.err+}+func (f *fakeIndex) Resolve(context.Context, string, string, string) (string, bool, error) {+ return f.subject, f.subject != "", f.err+}+func (f *fakeIndex) Complete(context.Context, string, string) (bool, error) { return f.complete, f.err }+func (f *fakeIndex) Allow(context.Context, string) (bool, error) { return !f.limited, f.err }+func (f *fakeIndex) Audit(_ context.Context, actor, outcome string) error {+ f.outcomes = append(f.outcomes, actor+":"+outcome)+ return f.auditErr+}++type fakeDirectory struct {+ account *keycloak.Account+ err error+ calls int+}++type privateLookupError struct{}++func (*privateLookupError) Error() string { return "private upstream detail" }++func TestFinishPreservesIncomingAndAuditErrors(t *testing.T) {+ for _, auditFails := range []bool{false, true} {+ cause := &privateLookupError{}+ auditCause := errors.New("private audit detail")+ index := &fakeIndex{}+ if auditFails {+ index.auditErr = auditCause+ }+ s := testService(t, index, &fakeDirectory{})+ _, err := s.finish(context.Background(), "actor", "unavailable", Result{}, cause)+ var typed *privateLookupError+ if !errors.Is(err, cause) || !errors.As(err, &typed) || typed != cause {+ t.Fatal("finish discarded incoming error")+ }+ if auditFails {+ if err.Error() != ErrUnavailable.Error() || !errors.Is(err, ErrUnavailable) || !errors.Is(err, auditCause) || errors.Unwrap(err) == nil {+ t.Fatal("audit failure must redact text and preserve both causes")+ }+ } else if err != cause {+ t.Fatal("successful audit must return incoming error unchanged")+ }+ }+}++func TestLookupPreservesPrivateCause(t *testing.T) {+ cause := &privateLookupError{}+ for _, fromDirectory := range []bool{false, true} {+ index, directory := &fakeIndex{}, &fakeDirectory{}+ if fromDirectory {+ directory.err = cause+ } else {+ index.err = cause+ }+ s := testService(t, index, directory)+ result, err := s.Lookup(context.Background(), "actor", Request{"account_id", "test-account"})+ var typed *privateLookupError+ if result.Account != nil || err == nil || err.Error() != ErrUnavailable.Error() || !errors.Is(err, ErrUnavailable) || !errors.Is(err, cause) || !errors.As(err, &typed) || typed != cause || errors.Unwrap(err) == nil {+ t.Fatal("lookup must redact text and preserve classification and cause")+ }+ }+}++func (f *fakeDirectory) LookupAccount(context.Context, string) (*keycloak.Account, error) {+ f.calls+++ return f.account, f.err+}+func testService(t *testing.T, index *fakeIndex, directory *fakeDirectory) *Service {+ t.Helper()+ ctx, cancel := context.WithCancel(context.Background())+ s := New(ctx, index, directory, "test-realm", "synthetic-test-key", []string{"excluded-account"})+ t.Cleanup(func() { cancel(); s.Wait() })+ return s+}++func TestLookupExactMatchAndCurrentClassification(t *testing.T) {+ index := &fakeIndex{subject: "test-account"}+ directory := &fakeDirectory{account: &keycloak.Account{ID: "test-account", Email: "[email protected]", EmailVerified: true}}+ s := testService(t, index, directory)+ pseudo := productanalytics.PseudonymForUserID("test-account", "synthetic-test-key")+ for _, input := range []Request{{"pseudonym", pseudo}, {"account_id", "test-account"}} {+ got, err := s.Lookup(context.Background(), "test-admin", input)+ if err != nil || got.Status != "found" || got.Account.ID != "test-account" || got.Account.Pseudonym != pseudo || got.Account.IdentityClass != productanalytics.IdentityInternal {+ t.Fatalf("unexpected exact lookup: %+v %v", got, err)+ }+ }+ if directory.calls != 2 {+ t.Fatal("lookup must use one exact directory call")+ }+ if strings.Contains(strings.Join(index.outcomes, " "), "test-account") {+ t.Fatal("target leaked into audit")+ }+ directory.account.EmailVerified = false+ got, err := s.Lookup(context.Background(), "test-admin", Request{"account_id", "test-account"})+ if err != nil || got.Account.IdentityClass != productanalytics.IdentityExternal {+ t.Fatal("unverified domain must not imply internal")+ }+}++func TestLookupFailuresDoNotDiscloseAccounts(t *testing.T) {+ for _, tc := range []struct {+ name string+ index fakeIndex+ dir fakeDirectory+ input Request+ status string+ err error+ calls int+ }{+ {"missing mapping", fakeIndex{}, fakeDirectory{}, Request{"pseudonym", "u_" + strings.Repeat("0", 64)}, "mapping_missing", nil, 0},+ {"scanned missing mapping", fakeIndex{complete: true}, fakeDirectory{}, Request{"pseudonym", "u_" + strings.Repeat("0", 64)}, "mapping_missing", nil, 0},+ {"deleted", fakeIndex{}, fakeDirectory{}, Request{"account_id", "deleted"}, "account_missing", nil, 1},+ {"directory down", fakeIndex{}, fakeDirectory{err: errors.New("sensitive upstream detail")}, Request{"account_id", "test-account"}, "", ErrUnavailable, 1},+ {"index down", fakeIndex{err: errors.New("db secret")}, fakeDirectory{}, Request{"account_id", "test-account"}, "", ErrUnavailable, 0},+ {"audit down", fakeIndex{auditErr: errors.New("audit down")}, fakeDirectory{account: &keycloak.Account{ID: "test-account"}}, Request{"account_id", "test-account"}, "", ErrUnavailable, 1},+ {"limited", fakeIndex{limited: true}, fakeDirectory{}, Request{"account_id", "test-account"}, "", ErrRateLimited, 0},+ {"bad kind", fakeIndex{}, fakeDirectory{}, Request{"email", "[email protected]"}, "", ErrInvalid, 0},+ {"traversal", fakeIndex{}, fakeDirectory{}, Request{"account_id", "../users"}, "", ErrInvalid, 0},+ {"oversize", fakeIndex{}, fakeDirectory{}, Request{"account_id", strings.Repeat("a", 256)}, "", ErrInvalid, 0},+ {"corrupt index", fakeIndex{subject: "wrong"}, fakeDirectory{}, Request{"pseudonym", "u_" + strings.Repeat("0", 64)}, "", ErrUnavailable, 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ s := testService(t, &tc.index, &tc.dir)+ got, err := s.Lookup(context.Background(), "test-admin", tc.input)+ if !errors.Is(err, tc.err) || got.Status != tc.status || got.Account != nil || tc.dir.calls != tc.calls {+ t.Fatalf("result=%+v err=%v calls=%d", got, err, tc.dir.calls)+ }+ })+ }+}++func TestObserverNormalizesHumanAndUserKeyIdentity(t *testing.T) {+ for _, principal := range []struct {+ kind, azp string+ want bool+ }{+ {auth.PrincipalTypeUser, "cyclops-cs-spa", true}, {auth.PrincipalTypeUser, "cua-cli", true},+ {auth.PrincipalTypeUserKey, "ukey-synthetic", true}, {auth.PrincipalTypeUser, "key-synthetic", false},+ {auth.PrincipalTypeUserKey, "custom-user-key-synthetic", true},+ {auth.PrincipalTypeGitHubOIDC, "github-actions", false},+ } {+ t.Run(principal.azp, func(t *testing.T) {+ index := &fakeIndex{recorded: make(chan string, 1)}+ s := testService(t, index, &fakeDirectory{})+ h := s.Observe(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) }))+ r := httptest.NewRequest("GET", "/api/config", nil)+ r = r.WithContext(context.WithValue(r.Context(), auth.UserKey, &auth.User{ID: "same-owner", AZP: principal.azp, PrincipalType: principal.kind}))+ w := httptest.NewRecorder()+ h.ServeHTTP(w, r)+ if w.Code != 204 {+ t.Fatal(w.Code)+ }+ if principal.want {+ select {+ case got := <-index.recorded:+ if got != productanalytics.PseudonymForUserID("same-owner", "synthetic-test-key")+":same-owner" {+ t.Fatal("identity changed")+ }+ case <-time.After(time.Second):+ t.Fatal("mapping not recorded")+ }+ } else if len(s.queue) != 0 || len(index.recorded) != 0 {+ t.Fatal("machine principal observed")+ }+ })+ }+}++func TestObserverFullQueueDoesNotBlock(t *testing.T) {+ s := &Service{key: "test", queue: make(chan string, 1)}+ s.queue <- "full"+ r := httptest.NewRequest("GET", "/api/config", nil)+ r = r.WithContext(context.WithValue(r.Context(), auth.UserKey, &auth.User{ID: "owner", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}))+ done := make(chan struct{})+ go func() {+ s.Observe(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})).ServeHTTP(httptest.NewRecorder(), r)+ close(done)+ }()+ select {+ case <-done:+ case <-time.After(time.Second):+ t.Fatal("observer blocked request")+ }+}diff --git a/libs/fleet/backend/accountlookup/store.go b/libs/fleet/backend/accountlookup/store.gonew file mode 100644index 0000000000..fd326b8d78--- /dev/null+++ b/libs/fleet/backend/accountlookup/store.go@@ -0,0 +1,121 @@+package accountlookup++import (+ "context"+ "errors"++ "cyclops-cs-backend/internal/redactederror"++ "github.com/jackc/pgx/v5"+ "github.com/jackc/pgx/v5/pgxpool"+)++type Store struct{ pool *pgxpool.Pool }++// NewStore is deliberately lazy: database availability must not gate startup.+func NewStore(ctx context.Context, dsn string) (*Store, error) {+ if dsn == "" {+ return nil, errors.New("account lookup database is not configured")+ }+ config, err := pgxpool.ParseConfig(dsn)+ if err != nil {+ return nil, redactederror.New("invalid account lookup database configuration", err)+ }+ config.MaxConns = 4+ pool, err := pgxpool.NewWithConfig(ctx, config)+ if err != nil {+ return nil, redactederror.New("invalid account lookup database configuration", err)+ }+ return &Store{pool: pool}, nil+}++func (s *Store) Close() { s.pool.Close() }++func (s *Store) Record(ctx context.Context, realm, keyID, pseudonym, subject string) error {+ tag, err := s.pool.Exec(ctx, `INSERT INTO account_lookup_private.mapping (realm,key_id,pseudonym,subject)+ VALUES ($1,$2,$3,$4) ON CONFLICT DO NOTHING`, realm, keyID, pseudonym, subject)+ if err != nil {+ return redactederror.New("account lookup record unavailable", err)+ }+ if tag.RowsAffected() == 0 {+ existing, found, err := s.Resolve(ctx, realm, keyID, pseudonym)+ if err != nil {+ return err+ }+ if !found || existing != subject {+ return errors.New("account lookup mapping conflict")+ }+ }+ return nil+}++func (s *Store) Resolve(ctx context.Context, realm, keyID, pseudonym string) (string, bool, error) {+ var subject string+ err := s.pool.QueryRow(ctx, `SELECT subject FROM account_lookup_private.mapping WHERE realm=$1 AND key_id=$2 AND pseudonym=$3`, realm, keyID, pseudonym).Scan(&subject)+ if errors.Is(err, pgx.ErrNoRows) {+ return "", false, nil+ }+ if err != nil {+ return "", false, redactederror.New("account lookup unavailable", err)+ }+ return subject, true, nil+}++func (s *Store) Complete(ctx context.Context, realm, keyID string) (bool, error) {+ var complete bool+ err := s.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM account_lookup_private.backfill WHERE realm=$1 AND key_id=$2)`, realm, keyID).Scan(&complete)+ if err != nil {+ return false, redactederror.New("account lookup scan status unavailable", err)+ }+ return complete, nil+}++// MarkScanComplete is called only after an uninterrupted scan starting at zero.+func (s *Store) MarkScanComplete(ctx context.Context, realm, keyID string) error {+ _, err := s.pool.Exec(ctx, `INSERT INTO account_lookup_private.backfill (realm,key_id) VALUES ($1,$2)+ ON CONFLICT (realm,key_id) DO UPDATE SET observed_scan_completed_at=clock_timestamp()`, realm, keyID)+ if err != nil {+ return redactederror.New("account lookup scan status unavailable", err)+ }+ return nil+}++func (s *Store) Audit(ctx context.Context, actor, outcome string) error {+ _, err := s.pool.Exec(ctx, `INSERT INTO account_lookup_private.audit (actor,outcome) VALUES ($1,$2)`, actor, outcome)+ if err != nil {+ return redactederror.New("account lookup audit unavailable", err)+ }+ return nil+}++// Allow serializes each actor across replicas and enforces a rolling minute.+func (s *Store) Allow(ctx context.Context, actor string) (bool, error) {+ tx, err := s.pool.Begin(ctx)+ if err != nil {+ return false, redactederror.New("account lookup rate limit unavailable", err)+ }+ defer tx.Rollback(ctx)+ if _, err = tx.Exec(ctx, `INSERT INTO account_lookup_private.rate_limit (actor) VALUES ($1) ON CONFLICT DO NOTHING`, actor); err != nil {+ return false, redactederror.New("account lookup rate limit unavailable", err)+ }+ var locked string+ if err = tx.QueryRow(ctx, `SELECT actor FROM account_lookup_private.rate_limit WHERE actor=$1 FOR UPDATE`, actor).Scan(&locked); err != nil {+ return false, redactederror.New("account lookup rate limit unavailable", err)+ }+ var allowed bool+ err = tx.QueryRow(ctx, `WITH recent AS (+ SELECT ARRAY(SELECT t FROM unnest(attempts) t WHERE t > clock_timestamp()-interval '1 minute') AS times+ FROM account_lookup_private.rate_limit WHERE actor=$1+ ) UPDATE account_lookup_private.rate_limit SET attempts=recent.times || clock_timestamp()+ FROM recent WHERE actor=$1 AND cardinality(recent.times)<10 RETURNING true`, actor).Scan(&allowed)+ if errors.Is(err, pgx.ErrNoRows) {+ err = nil+ }+ if err != nil {+ return false, redactederror.New("account lookup rate limit unavailable", err)+ }+ if err = tx.Commit(ctx); err != nil {+ return false, redactederror.New("account lookup rate limit unavailable", err)+ }+ return allowed, nil+}diff --git a/libs/fleet/backend/accountlookup/store_test.go b/libs/fleet/backend/accountlookup/store_test.gonew file mode 100644index 0000000000..e31a99363c--- /dev/null+++ b/libs/fleet/backend/accountlookup/store_test.go@@ -0,0 +1,35 @@+package accountlookup++import (+ "context"+ "errors"+ "strings"+ "testing"++ "github.com/jackc/pgx/v5/pgconn"+)++func TestStoreConstructionDoesNotRequireDatabase(t *testing.T) {+ store, err := NewStore(context.Background(), "postgres://test:[email protected]:1/test?connect_timeout=1")+ if err != nil {+ t.Fatal(err)+ }+ defer store.Close()+ if _, _, err := store.Resolve(context.Background(), "realm", "key", "user_test"); err == nil {+ t.Fatal("expected unavailable database error")+ }+}+func TestStoreConfigurationErrorsAreSanitized(t *testing.T) {+ for _, dsn := range []string{"", "://secret-value"} {+ _, err := NewStore(context.Background(), dsn)+ if err == nil || strings.Contains(err.Error(), "secret-value") {+ t.Fatalf("unexpected error: %v", err)+ }+ if dsn != "" {+ var parseErr *pgconn.ParseConfigError+ if errors.Unwrap(err) == nil || !errors.As(err, &parseErr) || !errors.Is(err, parseErr) {+ t.Fatal("configuration error must preserve its parser cause")+ }+ }+ }+}diff --git a/libs/fleet/backend/auth/account_lookup_test.go b/libs/fleet/backend/auth/account_lookup_test.gonew file mode 100644index 0000000000..b5cd0b1f13--- /dev/null+++ b/libs/fleet/backend/auth/account_lookup_test.go@@ -0,0 +1,53 @@+package auth++import (+ "context"+ "net/http"+ "net/http/httptest"+ "testing"+)++func TestAccountLookupFreshAuthorization(t *testing.T) {+ LoadOpa()+ original := computeFreshAdminFlagsFn+ t.Cleanup(func() { computeFreshAdminFlagsFn = original })+ members := []interface{}{"test-admin"}+ calls := 0+ computeFreshAdminFlagsFn = func(context.Context) map[string]interface{} {+ calls+++ return map[string]interface{}{"admin_subs": members}+ }+ policy := surfacePolicies["account-lookup"]+ h := PolicyMiddleware(policy.tree(), policy.options...)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ ok, err := EvalIsAdminFresh(r.Context(), GetUser(r.Context()))+ if err != nil || !ok {+ t.Fatal("fresh handler check failed")+ }+ w.WriteHeader(204)+ }))+ for _, tc := range []struct {+ name, azp, principal string+ revoke bool+ want int+ }{+ {"admin", "cyclops-cs-spa", PrincipalTypeUser, false, 204},+ {"CLI", "cua-cli", PrincipalTypeUser, false, 403},+ {"user key", "ukey-test", PrincipalTypeUserKey, false, 403},+ {"automation", "cyclops-cs-spa", PrincipalTypeGitHubOIDC, false, 403},+ {"revoked", "cyclops-cs-spa", PrincipalTypeUser, true, 403},+ } {+ t.Run(tc.name, func(t *testing.T) {+ if tc.revoke {+ members = []interface{}{}+ }+ r := routeRequest("POST", "/api/admin/account-lookup", "/api/admin/account-lookup", nil, "")+ r = r.WithContext(context.WithValue(r.Context(), UserKey, &User{ID: "test-admin", AZP: tc.azp, PrincipalType: tc.principal}))+ before := calls+ w := httptest.NewRecorder()+ h.ServeHTTP(w, r)+ if w.Code != tc.want || calls != before+1 {+ t.Fatalf("status=%d fresh calls=%d", w.Code, calls-before)+ }+ })+ }+}diff --git a/libs/fleet/backend/auth/authz_account_lookup.rego b/libs/fleet/backend/auth/authz_account_lookup.regonew file mode 100644index 0000000000..fc319c0ffb--- /dev/null+++ b/libs/fleet/backend/auth/authz_account_lookup.rego@@ -0,0 +1,11 @@+package authz_account_lookup++import data.authz++allow {+ input.route == "/api/admin/account-lookup"+ input.method == "POST"+ input.user.azp == "cyclops-cs-spa"+ input.user.principal_type == "user"+ authz.is_admin+}diff --git a/libs/fleet/backend/auth/middlewares.go b/libs/fleet/backend/auth/middlewares.goindex c0af4ad90a..0ed5e7f4bd 100644--- a/libs/fleet/backend/auth/middlewares.go+++ b/libs/fleet/backend/auth/middlewares.go@@ -83,8 +83,12 @@ var surfacePolicySources = map[string]struct {"authz-signed-service-urls": {"authz_signed_service_urls.rego", authzSignedServiceURLsPolicy},"authz-state-query": {"authz_state_query.rego", authzStateQueryPolicy},"authz-feature-flags": {"authz_feature_flags.rego", authzFeatureFlagsPolicy},+ "authz-account-lookup": {"authz_account_lookup.rego", authzAccountLookupPolicy},}+//go:embed authz_account_lookup.rego+var authzAccountLookupPolicy string+//go:embed authz_base.regovar authzBasePolicy stringdiff --git a/libs/fleet/backend/auth/policy_characterization_test.go b/libs/fleet/backend/auth/policy_characterization_test.goindex 61214ebf55..0fdd26a96b 100644--- a/libs/fleet/backend/auth/policy_characterization_test.go+++ b/libs/fleet/backend/auth/policy_characterization_test.go@@ -190,6 +190,7 @@ func characterizationCases() map[string][]routeCase {simple("/api/user-keys", "/api/user-keys")simple("/api/github-trust-policies", "/api/github-trust-policies")simple("/api/admin/feature-flags", "/api/admin/feature-flags")+ simple("/api/admin/account-lookup", "/api/admin/account-lookup")cases["/api/admin/feature-flags/{key}"] = []routeCase{{name: "key", params: map[string]string{"key": "example"}, path: "/api/admin/feature-flags/example"}}withID := func(route, prefix string) {diff --git a/libs/fleet/backend/auth/policy_routes.go b/libs/fleet/backend/auth/policy_routes.goindex fc21137d4c..c78f530499 100644--- a/libs/fleet/backend/auth/policy_routes.go+++ b/libs/fleet/backend/auth/policy_routes.go@@ -181,6 +181,10 @@ func FeatureFlagsRoutePolicy() Node {return All(BasePolicy(), surfaceLeaf("authz-feature-flags", "data.authz_feature_flags.allow"))}+func AccountLookupRoutePolicy() Node {+ return All(BasePolicy(), surfaceLeaf("authz-account-lookup", "data.authz_account_lookup.allow"))+}+// K8sRoutePolicy guards /api/k8s/{path...}. It is the same base + surface shape// as every other route, with two admission conjuncts: card-or-admin admission// for custom-resource creation, and pool admission over the request body.@@ -259,6 +263,7 @@ const featureFlagAuditBodyLimit = 64 << 10// surfacePolicies is every surface, by name. A surface owning no route fails// TestEveryPolicySurfaceOwnsARoute; a route naming no surface cannot start.var surfacePolicies = map[string]surfacePolicy{+ "account-lookup": {tree: AccountLookupRoutePolicy, options: []MiddlewareOption{WithAdminAPIErrorResponses(), WithFreshAdminAuthorization()}},"keys": {tree: KeysRoutePolicy},"config": {tree: ConfigRoutePolicy},"chat": {tree: ChatRoutePolicy},@@ -324,6 +329,7 @@ var routeSurfaces = map[string]string{"/api/k8s/{path...}": "k8s","/api/admin/feature-flags": "feature-flags",+ "/api/admin/account-lookup": "account-lookup","/api/admin/feature-flags/{key}": "feature-flags",}diff --git a/libs/fleet/backend/auth/testdata/account-lookup-route-plan.txt b/libs/fleet/backend/auth/testdata/account-lookup-route-plan.txtnew file mode 100644index 0000000000..b3dccdc479--- /dev/null+++ b/libs/fleet/backend/auth/testdata/account-lookup-route-plan.txt@@ -0,0 +1,3 @@+All+ Leaf query=data.authz_base.allow source=registered:authz-base+ Leaf query=data.authz_account_lookup.allow source=multi["registered:authz","registered:authz-account-lookup"]diff --git a/libs/fleet/backend/auth/testdata/route-authorization-table.txt b/libs/fleet/backend/auth/testdata/route-authorization-table.txtindex f0574e32e3..e7663d33d7 100644--- a/libs/fleet/backend/auth/testdata/route-authorization-table.txt+++ b/libs/fleet/backend/auth/testdata/route-authorization-table.txt@@ -1,3 +1,58 @@+/api/admin/account-lookup | plain | DELETE | admin-spa = deny+/api/admin/account-lookup | plain | DELETE | cua-cli = deny+/api/admin/account-lookup | plain | DELETE | dcr-client = deny+/api/admin/account-lookup | plain | DELETE | empty-sub = deny+/api/admin/account-lookup | plain | DELETE | github-oidc = deny+/api/admin/account-lookup | plain | DELETE | oauth2-proxy = deny+/api/admin/account-lookup | plain | DELETE | per-key-other-ns = deny+/api/admin/account-lookup | plain | DELETE | per-key-owned-ns = deny+/api/admin/account-lookup | plain | DELETE | spa = deny+/api/admin/account-lookup | plain | DELETE | user-key = deny+/api/admin/account-lookup | plain | DELETE | user-key-verified = deny+/api/admin/account-lookup | plain | GET | admin-spa = deny+/api/admin/account-lookup | plain | GET | cua-cli = deny+/api/admin/account-lookup | plain | GET | dcr-client = deny+/api/admin/account-lookup | plain | GET | empty-sub = deny+/api/admin/account-lookup | plain | GET | github-oidc = deny+/api/admin/account-lookup | plain | GET | oauth2-proxy = deny+/api/admin/account-lookup | plain | GET | per-key-other-ns = deny+/api/admin/account-lookup | plain | GET | per-key-owned-ns = deny+/api/admin/account-lookup | plain | GET | spa = deny+/api/admin/account-lookup | plain | GET | user-key = deny+/api/admin/account-lookup | plain | GET | user-key-verified = deny+/api/admin/account-lookup | plain | PATCH | admin-spa = deny+/api/admin/account-lookup | plain | PATCH | cua-cli = deny+/api/admin/account-lookup | plain | PATCH | dcr-client = deny+/api/admin/account-lookup | plain | PATCH | empty-sub = deny+/api/admin/account-lookup | plain | PATCH | github-oidc = deny+/api/admin/account-lookup | plain | PATCH | oauth2-proxy = deny+/api/admin/account-lookup | plain | PATCH | per-key-other-ns = deny+/api/admin/account-lookup | plain | PATCH | per-key-owned-ns = deny+/api/admin/account-lookup | plain | PATCH | spa = deny+/api/admin/account-lookup | plain | PATCH | user-key = deny+/api/admin/account-lookup | plain | PATCH | user-key-verified = deny+/api/admin/account-lookup | plain | POST | admin-spa = allow+/api/admin/account-lookup | plain | POST | cua-cli = deny+/api/admin/account-lookup | plain | POST | dcr-client = deny+/api/admin/account-lookup | plain | POST | empty-sub = deny+/api/admin/account-lookup | plain | POST | github-oidc = deny+/api/admin/account-lookup | plain | POST | oauth2-proxy = deny+/api/admin/account-lookup | plain | POST | per-key-other-ns = deny+/api/admin/account-lookup | plain | POST | per-key-owned-ns = deny+/api/admin/account-lookup | plain | POST | spa = deny+/api/admin/account-lookup | plain | POST | user-key = deny+/api/admin/account-lookup | plain | POST | user-key-verified = deny+/api/admin/account-lookup | plain | QUERY | admin-spa = deny+/api/admin/account-lookup | plain | QUERY | cua-cli = deny+/api/admin/account-lookup | plain | QUERY | dcr-client = deny+/api/admin/account-lookup | plain | QUERY | empty-sub = deny+/api/admin/account-lookup | plain | QUERY | github-oidc = deny+/api/admin/account-lookup | plain | QUERY | oauth2-proxy = deny+/api/admin/account-lookup | plain | QUERY | per-key-other-ns = deny+/api/admin/account-lookup | plain | QUERY | per-key-owned-ns = deny+/api/admin/account-lookup | plain | QUERY | spa = deny+/api/admin/account-lookup | plain | QUERY | user-key = deny+/api/admin/account-lookup | plain | QUERY | user-key-verified = deny/api/admin/feature-flags | plain | DELETE | admin-spa = allow/api/admin/feature-flags | plain | DELETE | cua-cli = deny/api/admin/feature-flags | plain | DELETE | dcr-client = denydiff --git a/libs/fleet/backend/cmd/account-lookup-backfill/main.go b/libs/fleet/backend/cmd/account-lookup-backfill/main.gonew file mode 100644index 0000000000..02b7f57da6--- /dev/null+++ b/libs/fleet/backend/cmd/account-lookup-backfill/main.go@@ -0,0 +1,102 @@+// account-lookup-backfill performs an explicitly authorized, bounded account scan.+package main++import (+ "context"+ "errors"+ "flag"+ "fmt"+ "os"+ "os/signal"+ "strings"+ "time"++ "cyclops-cs-backend/accountlookup"+ "cyclops-cs-backend/internal/redactederror"+ "cyclops-cs-backend/keycloak"+ "cyclops-cs-backend/productanalytics"+)++type accountLister interface {+ ListAccountIDs(context.Context, int, int) ([]string, error)+}+type mappingStore interface {+ Record(context.Context, string, string, string, string) error+ MarkScanComplete(context.Context, string, string) error+}++func scan(ctx context.Context, source accountLister, store mappingStore, realm, key string, offset, pageSize, maxPages int, execute bool) (int, bool, error) {+ count := 0+ for page := 0; page < maxPages; page++ {+ ids, err := source.ListAccountIDs(ctx, offset+count, pageSize)+ if err != nil {+ return count, false, redactederror.New("account scan failed", err)+ }+ if len(ids) > pageSize {+ return count, false, errors.New("account scan exceeded page bound")+ }+ for _, id := range ids {+ if strings.TrimSpace(id) == "" {+ return count, false, errors.New("account scan returned invalid account")+ }+ if execute {+ if err := store.Record(ctx, realm, accountlookup.KeyID(key), productanalytics.PseudonymForUserID(id, key), id); err != nil {+ return count, false, redactederror.New("account mapping write failed", err)+ }+ }+ count+++ }+ if len(ids) < pageSize {+ if execute && offset == 0 {+ if err := store.MarkScanComplete(ctx, realm, accountlookup.KeyID(key)); err != nil {+ return count, false, redactederror.New("account scan completion write failed", err)+ }+ }+ return count, true, nil+ }+ }+ return count, false, nil+}++func run() error {+ execute := flag.Bool("execute", false, "write mappings; default only reads Keycloak")+ offset := flag.Int("offset", 0, "resume offset (never marks complete)")+ pageSize := flag.Int("page-size", 100, "accounts per page (1-100)")+ maxPages := flag.Int("max-pages", 100, "maximum pages this invocation (1-10000)")+ flag.Parse()+ if flag.NArg() != 0 || *offset < 0 || *offset > 1000000000 || *pageSize < 1 || *pageSize > 100 || *maxPages < 1 || *maxPages > 10000 {+ return errors.New("invalid backfill bounds")+ }+ key := strings.TrimSpace(os.Getenv("POSTHOG_IDENTITY_KEY"))+ realm := strings.TrimSpace(os.Getenv("KC_REALM"))+ baseURL := strings.TrimSpace(os.Getenv("KC_BASE_URL"))+ clientID := strings.TrimSpace(os.Getenv("KC_ADMIN_CLIENT_ID"))+ secret := os.Getenv("KC_ADMIN_CLIENT_SECRET")+ if key == "" || realm == "" || baseURL == "" || clientID == "" || secret == "" {+ return errors.New("explicit Keycloak configuration and POSTHOG_IDENTITY_KEY are required")+ }+ ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)+ defer cancel()+ ctx, deadline := context.WithTimeout(ctx, 30*time.Minute)+ defer deadline()+ var store *accountlookup.Store+ if *execute {+ var err error+ store, err = accountlookup.NewStore(ctx, os.Getenv("DATABASE_URL"))+ if err != nil {+ return redactederror.New("backfill database configuration invalid", err)+ }+ defer store.Close()+ }+ source := keycloak.NewAdmin(baseURL, realm, clientID, secret, "", "")+ count, end, err := scan(ctx, source, store, realm, key, *offset, *pageSize, *maxPages, *execute)+ fmt.Printf("execute=%t scanned=%d next_offset=%d observed_end=%t completion_eligible=%t\n", *execute, count, *offset+count, end, *execute && *offset == 0 && end && err == nil)+ return err+}++func main() {+ if err := run(); err != nil {+ fmt.Fprintln(os.Stderr, err)+ os.Exit(1)+ }+}diff --git a/libs/fleet/backend/cmd/account-lookup-backfill/main_test.go b/libs/fleet/backend/cmd/account-lookup-backfill/main_test.gonew file mode 100644index 0000000000..d76b4618b0--- /dev/null+++ b/libs/fleet/backend/cmd/account-lookup-backfill/main_test.go@@ -0,0 +1,95 @@+package main++import (+ "context"+ "errors"+ "testing"+)++type fakeSource struct {+ pages [][]string+ fail bool+ err error+}++func (s *fakeSource) ListAccountIDs(context.Context, int, int) ([]string, error) {+ if err := s.err; err != nil {+ return nil, err+ }+ if s.fail {+ return nil, errors.New("private upstream details")+ }+ page := s.pages[0]+ s.pages = s.pages[1:]+ return page, nil+}++type fakeStore struct {+ recorded, completed int+ recordErr, completeErr error+}++func (s *fakeStore) Record(context.Context, string, string, string, string) error {+ s.recorded+++ return s.recordErr+}+func (s *fakeStore) MarkScanComplete(context.Context, string, string) error {+ s.completed+++ return s.completeErr+}++type privateScanError struct{}++func (*privateScanError) Error() string { return "private upstream details" }++func TestScanPreservesRedactedCauses(t *testing.T) {+ cause := &privateScanError{}+ for _, tc := range []struct {+ name string+ source fakeSource+ store fakeStore+ message string+ }{+ {"read", fakeSource{err: cause}, fakeStore{}, "account scan failed"},+ {"record", fakeSource{pages: [][]string{{"one"}}}, fakeStore{recordErr: cause}, "account mapping write failed"},+ {"complete", fakeSource{pages: [][]string{{}}}, fakeStore{completeErr: cause}, "account scan completion write failed"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ _, end, err := scan(context.Background(), &tc.source, &tc.store, "realm", "key", 0, 2, 2, true)+ var typed *privateScanError+ if end || err == nil || err.Error() != tc.message || !errors.Is(err, cause) || !errors.As(err, &typed) || typed != cause || errors.Unwrap(err) != cause {+ t.Fatal("failed scan must redact text while preserving the cause")+ }+ })+ }+}+func TestScanCompletionBoundaries(t *testing.T) {+ for _, tc := range []struct {+ name string+ offset, max int+ execute bool+ pages [][]string+ end bool+ recorded, completed int+ }{+ {"dry run", 0, 2, false, [][]string{{"one"}}, true, 0, 0},+ {"full observed scan", 0, 2, true, [][]string{{"one", "two"}, {}}, true, 2, 1},+ {"resume cannot complete", 2, 2, true, [][]string{{"three"}}, true, 1, 0},+ {"bounded cannot complete", 0, 1, true, [][]string{{"one", "two"}}, false, 2, 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ store := &fakeStore{}+ _, end, err := scan(context.Background(), &fakeSource{pages: tc.pages}, store, "realm", "key", tc.offset, 2, tc.max, tc.execute)+ if err != nil || end != tc.end || store.recorded != tc.recorded || store.completed != tc.completed {+ t.Fatalf("end=%t err=%v store=%+v", end, err, store)+ }+ })+ }+}+func TestFailedScanNeverCompletes(t *testing.T) {+ store := &fakeStore{}+ _, end, err := scan(context.Background(), &fakeSource{fail: true}, store, "realm", "key", 0, 2, 2, true)+ if err == nil || end || store.completed != 0 || err.Error() != "account scan failed" {+ t.Fatalf("end=%t err=%v", end, err)+ }+}diff --git a/libs/fleet/backend/database/account_lookup_pg_test.go b/libs/fleet/backend/database/account_lookup_pg_test.gonew file mode 100644index 0000000000..8f0251291a--- /dev/null+++ b/libs/fleet/backend/database/account_lookup_pg_test.go@@ -0,0 +1,178 @@+package database++import (+ "context"+ "errors"+ "sync"+ "sync/atomic"+ "testing"++ "cyclops-cs-backend/accountlookup"+ "cyclops-cs-backend/keycloak"+ "github.com/jackc/pgx/v5"+)++type fixtureAccountDirectory struct{}++func (fixtureAccountDirectory) LookupAccount(_ context.Context, id string) (*keycloak.Account, error) {+ return &keycloak.Account{ID: id}, nil+}++func TestAccountLookupUpgradeFromPreviousSchema(t *testing.T) {+ maintenanceURL := requireMigratorIntegration(t)+ ctx := context.Background()+ migrationURL, credentials := isolatedMigrationDatabase(t, ctx, maintenanceURL)+ files, err := embeddedMigrations()+ if err != nil {+ t.Fatal(err)+ }+ connection := connect(t, ctx, migrationURL)+ defer connection.Close(ctx)+ tx, err := connection.Begin(ctx)+ if err != nil {+ t.Fatal(err)+ }+ defer tx.Rollback(ctx)+ if err = newRuntimeDDL(tx).ensureMigrationLedger(ctx); err != nil {+ t.Fatal(err)+ }+ for _, file := range files[:11] {+ prepared, prepareErr := prepareMigrationExecution(file)+ if prepareErr != nil {+ t.Fatal(prepareErr)+ }+ if _, err = tx.Exec(ctx, prepared.SQL); err != nil {+ t.Fatal(err)+ }+ if _, err = tx.Exec(ctx, insertAppliedMigrationStatement, file.Version, file.Name, file.SHA256); err != nil {+ t.Fatal(err)+ }+ }+ if err = tx.Commit(ctx); err != nil {+ t.Fatal(err)+ }+ store, err := accountlookup.NewStore(ctx, migrationURL)+ if err != nil {+ t.Fatal(err)+ }+ defer store.Close()+ serviceCtx, cancel := context.WithCancel(ctx)+ service := accountlookup.New(serviceCtx, store, fixtureAccountDirectory{}, "test", "test", nil)+ defer func() { cancel(); service.Wait() }()+ if _, err = service.Lookup(ctx, "admin", accountlookup.Request{Kind: "account_id", Value: "test"}); !errors.Is(err, accountlookup.ErrUnavailable) {+ t.Fatal("pre-migration feature should be unavailable")+ }+ if _, _, err = store.Resolve(ctx, "test", "test", "test"); err == nil {+ t.Fatal("pre-migration table unexpectedly present")+ }+ for i := 0; i < 2; i++ {+ summary := captureRunSummary(t, func() error { return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials}) })+ wantPending := 1 - i+ if summary.Pending != wantPending || summary.Applied != wantPending || summary.Target != 12 {+ t.Fatalf("upgrade/no-op summary=%+v", summary)+ }+ }+ if _, _, err = store.Resolve(ctx, "test", "test", "test"); err != nil {+ t.Fatal("existing pool failed to recover after migration")+ }+ if result, err := service.Lookup(ctx, "admin", accountlookup.Request{Kind: "account_id", Value: "test"}); err != nil || result.Status != "found" {+ t.Fatal("lookup failed to recover after migration")+ }+}++func assertAccountLookupContract(t *testing.T, ctx context.Context, ownerURL, appURL, reportingURL, tenantURL string) {+ t.Helper()+ first, err := accountlookup.NewStore(ctx, appURL)+ if err != nil {+ t.Fatal(err)+ }+ defer first.Close()+ second, err := accountlookup.NewStore(ctx, appURL)+ if err != nil {+ t.Fatal(err)+ }+ defer second.Close()+ if err = first.Record(ctx, "realm", "key", "pseudonym", "subject"); err != nil {+ t.Fatal(err)+ }+ if err = first.Record(ctx, "realm", "key", "pseudonym", "subject"); err != nil {+ t.Fatal(err)+ }+ if err = first.Record(ctx, "realm", "key", "pseudonym", "other-subject"); err == nil {+ t.Fatal("mapping overwrite succeeded")+ }+ subject, found, err := second.Resolve(ctx, "realm", "key", "pseudonym")+ if err != nil || !found || subject != "subject" {+ t.Fatalf("mapping lookup failed: found=%t err=%v", found, err)+ }+ for _, scope := range [][2]string{{"other-realm", "key"}, {"realm", "other-key"}} {+ _, found, err = second.Resolve(ctx, scope[0], scope[1], "pseudonym")+ if err != nil || found {+ t.Fatalf("mapping crossed scope: found=%t err=%v", found, err)+ }+ }+ if err = first.MarkScanComplete(ctx, "realm", "key"); err != nil {+ t.Fatal(err)+ }+ for _, scope := range []struct {+ realm, key string+ want bool+ }{{"realm", "key", true}, {"other-realm", "key", false}, {"realm", "other-key", false}} {+ complete, err := second.Complete(ctx, scope.realm, scope.key)+ if err != nil || complete != scope.want {+ t.Fatalf("completion=%t err=%v", complete, err)+ }+ }+ if err = first.Audit(ctx, "actor", "found"); err != nil {+ t.Fatal(err)+ }+ if err = first.Audit(ctx, "actor", "arbitrary-target-text"); err == nil {+ t.Fatal("unbounded audit outcome accepted")+ }+ var allowed atomic.Int32+ var wg sync.WaitGroup+ for i := 0; i < 30; i++ {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ s := first+ if i%2 == 0 {+ s = second+ }+ ok, err := s.Allow(ctx, "actor")+ if err != nil {+ t.Error(err)+ }+ if ok {+ allowed.Add(1)+ }+ }(i)+ }+ wg.Wait()+ if allowed.Load() != 10 {+ t.Fatalf("allowed=%d want 10", allowed.Load())+ }+ owner, err := pgx.Connect(ctx, ownerURL)+ if err != nil {+ t.Fatal(err)+ }+ defer owner.Close(ctx)+ if _, err = owner.Exec(ctx, `UPDATE account_lookup_private.rate_limit SET attempts=ARRAY[clock_timestamp()-interval '61 seconds'] WHERE actor='actor'`); err != nil {+ t.Fatal(err)+ }+ if ok, err := first.Allow(ctx, "actor"); err != nil || !ok {+ t.Fatalf("expired window allowed=%t err=%v", ok, err)+ }+ var owned bool+ if err = owner.QueryRow(ctx, `SELECT nspowner=(SELECT oid FROM pg_roles WHERE rolname=current_user) FROM pg_namespace WHERE nspname='account_lookup_private'`).Scan(&owned); err != nil || !owned {+ t.Fatalf("schema ownership=%t err=%v", owned, err)+ }+ for _, url := range []string{reportingURL, tenantURL} {+ for _, table := range []string{"mapping", "audit", "backfill", "rate_limit"} {+ assertStatementFails(t, ctx, url, "SELECT * FROM account_lookup_private."+table)+ }+ }+ assertStatementFails(t, ctx, appURL, "SELECT * FROM account_lookup_private.audit")+ assertStatementFails(t, ctx, appURL, "DELETE FROM account_lookup_private.mapping")+ assertStatementFails(t, ctx, appURL, "CREATE TABLE account_lookup_private.unexpected (id int)")+}diff --git a/libs/fleet/backend/database/migrations/000012_private_account_lookup.sql b/libs/fleet/backend/database/migrations/000012_private_account_lookup.sqlnew file mode 100644index 0000000000..084293a3d5--- /dev/null+++ b/libs/fleet/backend/database/migrations/000012_private_account_lookup.sql@@ -0,0 +1,39 @@+CREATE SCHEMA account_lookup_private AUTHORIZATION CURRENT_USER;+REVOKE ALL ON SCHEMA account_lookup_private FROM PUBLIC;+GRANT USAGE ON SCHEMA account_lookup_private TO cyclops_app;++CREATE TABLE account_lookup_private.mapping (+ realm text NOT NULL CHECK (realm <> ''),+ key_id text NOT NULL CHECK (key_id <> ''),+ pseudonym text NOT NULL CHECK (pseudonym <> ''),+ subject text NOT NULL CHECK (subject <> ''),+ PRIMARY KEY (realm, key_id, pseudonym)+);++-- Completion means a successful observed scan from offset zero, not a snapshot.+CREATE TABLE account_lookup_private.backfill (+ realm text NOT NULL,+ key_id text NOT NULL,+ observed_scan_completed_at timestamptz NOT NULL DEFAULT clock_timestamp(),+ PRIMARY KEY (realm, key_id)+);++CREATE TABLE account_lookup_private.audit (+ actor text NOT NULL CHECK (actor <> ''),+ occurred_at timestamptz NOT NULL DEFAULT clock_timestamp(),+ outcome text NOT NULL CHECK (outcome IN (+ 'found', 'not_found', 'mapping_missing', 'account_missing',+ 'invalid', 'unavailable', 'rate_limited', 'forbidden'+ ))+);++CREATE TABLE account_lookup_private.rate_limit (+ actor text PRIMARY KEY CHECK (actor <> ''),+ attempts timestamptz[] NOT NULL DEFAULT '{}'+);++REVOKE ALL ON ALL TABLES IN SCHEMA account_lookup_private FROM PUBLIC;+GRANT SELECT, INSERT ON account_lookup_private.mapping TO cyclops_app;+GRANT SELECT, INSERT, UPDATE ON account_lookup_private.backfill TO cyclops_app;+GRANT INSERT ON account_lookup_private.audit TO cyclops_app;+GRANT SELECT, INSERT, UPDATE ON account_lookup_private.rate_limit TO cyclops_app;diff --git a/libs/fleet/backend/database/migrator_pg_test.go b/libs/fleet/backend/database/migrator_pg_test.goindex 42e1e4b9c9..391220d0c1 100644--- a/libs/fleet/backend/database/migrator_pg_test.go+++ b/libs/fleet/backend/database/migrator_pg_test.go@@ -91,8 +91,8 @@ func TestInitialMigrationBuildsCompleteDatabase(t *testing.T) {firstSummary := captureRunSummary(t, func() error {return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})})- if firstSummary.Pending != 11 || firstSummary.Applied != 11 {- t.Fatalf("initial migration summary = %+v, want pending=11 applied=11", firstSummary)+ if firstSummary.Pending != 12 || firstSummary.Applied != 12 {+ t.Fatalf("initial migration summary = %+v, want pending=12 applied=12", firstSummary)}before := migrationLedgerRows(t, ctx, migrationURL)secondSummary := captureRunSummary(t, func() error {@@ -136,6 +136,7 @@ func TestInitialMigrationBuildsCompleteDatabase(t *testing.T) {assertUsageReaderBoundary(t, ctx, inspectionURL, credentials.Usage)assertApplicationBoundary(t, ctx, credentials.Application)assertChatConversationStore(t, ctx, credentials.Application)+ assertAccountLookupContract(t, ctx, migrationURL, credentials.Application, credentials.Metabase, tenantURL)}func assertChatConversationStore(t *testing.T, ctx context.Context, applicationURL string) {@@ -200,14 +201,14 @@ func TestRunUpgradesVersionOneAndThenNoOps(t *testing.T) {upgrade := captureRunSummary(t, func() error {return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})})- if upgrade.Current != 1 || upgrade.Target != 11 || upgrade.Pending != 10 || upgrade.Applied != 10 || upgrade.Skipped != 1 || upgrade.Result != "success" {+ if upgrade.Current != 1 || upgrade.Target != 12 || upgrade.Pending != 11 || upgrade.Applied != 11 || upgrade.Skipped != 1 || upgrade.Result != "success" {t.Fatalf("version-one upgrade summary = %+v", upgrade)}noOp := captureRunSummary(t, func() error {return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})})- if noOp.Current != 11 || noOp.Target != 11 || noOp.Pending != 0 || noOp.Applied != 0 || noOp.Skipped != 11 || noOp.Result != "success" {+ if noOp.Current != 12 || noOp.Target != 12 || noOp.Pending != 0 || noOp.Applied != 0 || noOp.Skipped != 12 || noOp.Result != "success" {t.Fatalf("post-upgrade no-op summary = %+v", noOp)}}@@ -1603,8 +1604,8 @@ func migrationLedgerRows(t *testing.T, ctx context.Context, adminURL string) []lif err := rows.Err(); err != nil {t.Fatal("iterate migration ledger")}- if len(ledger) != 11 {- t.Fatalf("migration ledger row count = %d, want 11", len(ledger))+ if len(ledger) != 12 {+ t.Fatalf("migration ledger row count = %d, want 12", len(ledger))}for index, want := range []struct {version int64@@ -1621,6 +1622,7 @@ func migrationLedgerRows(t *testing.T, ctx context.Context, adminURL string) []l{9, "000009_extend_metabase_revenue_tenant_exclusions.sql"},{10, "000010_grant_metabase_billing_meter_access.sql"},{11, "000011_signed_service_urls.sql"},+ {12, "000012_private_account_lookup.sql"},} {if ledger[index].Version != want.version || ledger[index].ApplicationOrder != int64(index+1) || ledger[index].Filename != want.filename {t.Fatalf("migration ledger row %d = version:%d order:%d filename:%q", index, ledger[index].Version, ledger[index].ApplicationOrder, ledger[index].Filename)@@ -2092,7 +2094,7 @@ func assertRuntimeLedgerAccess(t *testing.T, ctx context.Context, credentials Crvar count interr := connection.QueryRow(ctx, `select count(*) from cyclops_migrations.applied_migrations`).Scan(&count)connection.Close(ctx)- if err != nil || count != 11 {+ if err != nil || count != 12 {t.Errorf("%s ledger select = count:%d err:%v", role, count, err)}assertStatementFails(t, ctx, databaseURL, `insert into cyclops_migrations.applied_migrations (version, filename, sha256) values (99, 'invalid.sql', 'invalid')`)diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.goindex f64925734f..6775c96913 100644--- a/libs/fleet/backend/database/migrator_test.go+++ b/libs/fleet/backend/database/migrator_test.go@@ -521,8 +521,8 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {if err != nil {t.Fatal(err)}- if len(files) != 11 {- t.Fatalf("expected exactly eleven migrations, got %d", len(files))+ if len(files) != 12 {+ t.Fatalf("expected exactly twelve migrations, got %d", len(files))}manifest := make([]struct {Version int64@@ -549,6 +549,7 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {{9, "000009_extend_metabase_revenue_tenant_exclusions.sql"},{10, "000010_grant_metabase_billing_meter_access.sql"},{11, "000011_signed_service_urls.sql"},+ {12, "000012_private_account_lookup.sql"},}) {t.Fatalf("migration manifest = %#v", manifest)}diff --git a/libs/fleet/backend/docs/account-lookup.md b/libs/fleet/backend/docs/account-lookup.mdnew file mode 100644index 0000000000..01d95e6f54--- /dev/null+++ b/libs/fleet/backend/docs/account-lookup.md@@ -0,0 +1,164 @@+# Private account lookup++The lookup index maps `(realm, identity-key fingerprint, pseudonym)` to a Keycloak+subject. It stores no email, username, or profile. The backend fetches current+account details from Keycloak only for an authorized exact lookup. Analytics+events continue using the existing `POSTHOG_IDENTITY_KEY` pseudonym.++Migration `000012_private_account_lookup.sql` creates a private schema owned by+the migration role. Only `cyclops_app` receives runtime schema access. Reporting+and tenant roles receive no access. Audit records contain actor, time, and a+fixed outcome only; they contain no lookup target or result. A PostgreSQL row+lock enforces ten admitted requests per actor in a rolling minute across replicas.++The API requires an authenticated `cyclops-cs-spa` human principal and a fresh+admin-membership decision for each request. CLI, API-key, proxy, and automation+principals cannot use it, even when their owner is an admin. Successful lookups+fail closed if the private audit cannot be written. Authenticated denials are+audited on a best-effort basis without reading the lookup body.++Future validated SPA, CLI, desktop, and normalized user-key identities enter a+bounded background queue, including analytics-excluded accounts. Successful+writes are deduplicated in memory for one hour. A full queue, database outage,+or process exit can lose observations; a later request or controlled backfill+can repair them. Neither authentication nor workload success depends on this+index. No new startup/readiness dependency is added. Before migration, the+lookup returns 503 while other routes keep serving.++## API contract++This feature is API-only; it adds no admin page or navigation entry.+Send `POST /api/admin/account-lookup` with `Content-Type: application/json` and+an `Authorization: Bearer` header containing a valid human admin access token+issued for `cyclops-cs-spa`. The HTTP caller does not need a custom UI, but CLI+client tokens, service accounts, and customer API keys remain denied. This+change does not add a token-issuance flow or expand client authorization.++For an exact account ID, send this synthetic example body:++```json+{"kind":"account_id","value":"synthetic-account-123"}+```++To resolve an analytics identity, set `kind` to `pseudonym` and `value` to the+exact `u_`-prefixed, 64-character lowercase hexadecimal HMAC value. Send the+identifier in the JSON body, not the URL. Requests are bounded to 1 KiB and+reject unknown fields. Email search, username search, fuzzy search, and bulk+lookup are not supported.++A successful HTTP response has `status`, `backfill_complete`, and, when found,+an `account` object:++| Status | Meaning |+| --- | --- |+| `found` | One current Keycloak account was retrieved. |+| `mapping_missing` | No mapping exists for that pseudonym and configured key. This does not prove the account never existed. |+| `account_missing` | Keycloak returned no current account for the exact subject. Historical events may still exist. |++The account contains `id`, `username`, `email`, `email_verified`, optional+`created_at`, `pseudonym`, `identity_class`, and `excluded`. Classification and+exclusion reflect current account/configuration evidence, not historical event+classification. The existing rule treats a verified `@trycua.com` email as+internal; other authenticated IDs default to external. That default is not+proof of customer ownership.++`backfill_complete` means an observed scan starting at offset zero reached the+end; it does not guarantee exhaustive coverage. The limitations are described+in the backfill section.++HTTP 400 indicates invalid input; 401/403 indicates missing or denied+authorization; 429 indicates the shared rate limit, with `Retry-After: 60`.+The lookup returns 503 when its database, directory, or required audit is+unavailable. Upstream authorization failures can also return a generic 5xx.+Responses use `Cache-Control: no-store`. Callers must not send input or returned+account details to analytics, public logs, shell history, or shared artifacts.++## Controlled historical backfill++Use an approved environment with the existing `POSTHOG_IDENTITY_KEY` and explicit+`KC_BASE_URL`, `KC_REALM`, `KC_ADMIN_CLIENT_ID`, and `KC_ADMIN_CLIENT_SECRET`.+Never paste credentials into command arguments or logs. Execute mode additionally+requires `DATABASE_URL` for the application role. This command intentionally+does not initialize telemetry or the serving application.++From `cyclops-cs/backend`, preview the bounded scan:++```sh+go run ./cmd/account-lookup-backfill --max-pages 100 --page-size 100+```++The default is read-only and prints counts and offsets only. After approval of+the target environment and scope, add `--execute` to write mappings. Repeated+runs are idempotent; a conflicting subject fails closed. `--offset N` resumes+work, but a resumed invocation never marks coverage complete. To establish the+observed scan marker, rerun from zero with enough `--max-pages` to reach a short+or empty final page. Interrupted, failed, dry-run, and capped scans do not create+a marker. Previously successful markers remain historical observations.++The marker means a scan starting at zero observed the end of Keycloak's paged+listing. The listing is not a snapshot: concurrent account changes can shift+offsets. It does not prove exhaustive current or historical coverage. Missing+mappings remain explicitly missing, even after a successful scan. Deleted users+cannot be reconstructed. Key rotation scopes new rows and scan status separately;+retain an approved key-to-history strategy before rotating the analytics key.++The backend release image includes `/app/account-lookup-backfill`. Operators can+run the same bounded command inside the deployed backend container with its+existing environment; no additional secret export or credential is required.+Run the read-only preview first and keep output limited to the command's counts.++## Rollout gates++1. Verify the additive migration on an isolated PostgreSQL cluster, including+ initial apply, immediate no-op, application behavior, and denied tenant and+ reporting access.+2. Deploy the migration through the normal immutable migration Job and verify+ its ledger and no-op result while serving remains available.+3. Verify authorization, current Keycloak read permissions, exact lookup, safe+ audit behavior, and rate limiting before enabling operator access.+4. Run the read-only backfill preview, approve the production target and scan bounds, then execute+ the backfill and verify observed scan status. A local test is not deployment+ or production backfill evidence.++Merge, deployment, production migration, backfill execution, and any access-grant+change each remain human-gated. The checked-in realm export declares user-read+roles for the backend client, but it is not evidence of current live grants.+No privilege expansion is part of this change.++Rollback the serving release to remove the endpoint, stop any backfill,+and leave the additive private schema in place. Do not delete or edit migration+12 or its ledger entry. Removing private data requires a separately approved+retention/deletion operation, not a schema rollback. The analytics key, event+contract, dashboard queries, and activation counts are unchanged.++The private tables are sensitive operational data. Apply the same restricted+backup and access policy as the account directory. Audit and inactive limiter+rows require an operator-defined retention policy; this change schedules no+automatic deletion.++## Local verification (September 4, 2026)++The implementation was tested in the supplied worktree based on `d1a5abee`.+No production secret, account directory, or database was used.++- Backend tests passed for `accountlookup`, `auth`, `handlers`, `keycloak`,+ `productanalytics`, the backfill command, and the main router package.+- Race checks passed for the lookup, handlers, Keycloak, and backfill packages.+- The full `database` suite passed against disposable loopback PostgreSQL 16.14,+ including version 11 to 12 (`pending=1`, `applied=1`), its immediate no-op,+ recovery after migration, and reporting/tenant denial assertions.+- Go vet, the serving-availability manifest contract, and whitespace checks passed.++The earlier UI, client wrapper, browser-only tests, and local visual preview+were removed when the scope changed to API-only. Their browser verification is+not evidence for the delivered API-only change. Backend tests exercise exact+lookup, strict request decoding, authorization and revocation, safe failures,+restricted auditing, and normalized identity mapping without a browser.++Changed-file scope: the `accountlookup` package and backfill command; append-only+migration 12 and database tests; Keycloak read wrappers; the lookup handler,+router wiring, OPA policy and authorization fixtures; and this API runbook.+The existing backend Dockerfile also packages the backfill binary.+No analytics event implementation, dashboard, deployment manifest, credential,+or existing migration was changed. The pre-existing `output/` directory was+left untouched.diff --git a/libs/fleet/backend/handlers/account_lookup.go b/libs/fleet/backend/handlers/account_lookup.gonew file mode 100644index 0000000000..0abab52382--- /dev/null+++ b/libs/fleet/backend/handlers/account_lookup.go@@ -0,0 +1,84 @@+package handlers++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "time"++ "cyclops-cs-backend/accountlookup"+ "cyclops-cs-backend/auth"+)++func PrivateAccountLookup(next http.Handler) http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Cache-Control", "no-store")+ w.Header().Set("Pragma", "no-cache")+ next.ServeHTTP(w, r)+ })+}++// Runs after token validation but before policy, so denied principals are audited+// without decoding their request bodies or copying them to ordinary logs.+func (h Handlers) AuditAccountLookupAccess(next http.Handler) http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ captured := &statusCapture{ResponseWriter: w}+ next.ServeHTTP(captured, r)+ if captured.statusCode == http.StatusForbidden {+ if u := currentUser(r); u != nil {+ ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), time.Second)+ defer cancel()+ h.AccountLookup.AuditDenied(ctx, u.ID)+ }+ }+ })+}++// LookupAccount discloses one account only to a freshly authorized human admin.+func (h Handlers) LookupAccount(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Cache-Control", "no-store")+ w.Header().Set("Pragma", "no-cache")+ u := currentUser(r)+ if u == nil || u.ID == "" || u.PrincipalType != auth.PrincipalTypeUser || u.AZP != "cyclops-cs-spa" {+ writeErr(w, http.StatusForbidden, "account lookup requires a human admin session")+ return+ }+ allowed, err := h.isAdmin(r.Context(), u)+ if err != nil {+ writeErr(w, http.StatusServiceUnavailable, "account lookup authorization unavailable")+ return+ }+ if !allowed {+ writeErr(w, http.StatusForbidden, "account lookup requires a human admin session")+ return+ }+ if r.Method != http.MethodPost {+ writeErr(w, http.StatusMethodNotAllowed, "POST required")+ return+ }+ var input accountlookup.Request+ d := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1024))+ d.DisallowUnknownFields()+ if d.Decode(&input) != nil || d.Decode(&struct{}{}) != io.EOF {+ // Invalid input is still rate limited and audited by the service.+ input = accountlookup.Request{}+ }+ ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)+ defer cancel()+ result, err := h.AccountLookup.Lookup(ctx, u.ID, input)+ switch {+ case errors.Is(err, accountlookup.ErrUnavailable):+ writeErr(w, http.StatusServiceUnavailable, "account lookup unavailable")+ case errors.Is(err, accountlookup.ErrInvalid):+ writeErr(w, http.StatusBadRequest, "provide one exact pseudonym or account ID")+ case errors.Is(err, accountlookup.ErrRateLimited):+ w.Header().Set("Retry-After", "60")+ writeErr(w, http.StatusTooManyRequests, "account lookup limit reached; try again in a minute")+ case err != nil:+ writeErr(w, http.StatusServiceUnavailable, "account lookup unavailable")+ default:+ writeJSON(w, http.StatusOK, result)+ }+}diff --git a/libs/fleet/backend/handlers/account_lookup_test.go b/libs/fleet/backend/handlers/account_lookup_test.gonew file mode 100644index 0000000000..76203d5d9e--- /dev/null+++ b/libs/fleet/backend/handlers/account_lookup_test.go@@ -0,0 +1,130 @@+package handlers++import (+ "context"+ "errors"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "cyclops-cs-backend/accountlookup"+ "cyclops-cs-backend/auth"+ "cyclops-cs-backend/keycloak"+)++type lookupTestIndex struct {+ outcomes []string+ auditErr error+}++func (*lookupTestIndex) Record(context.Context, string, string, string, string) error { return nil }+func (*lookupTestIndex) Resolve(context.Context, string, string, string) (string, bool, error) {+ return "", false, nil+}+func (*lookupTestIndex) Complete(context.Context, string, string) (bool, error) { return false, nil }+func (*lookupTestIndex) Allow(context.Context, string) (bool, error) { return true, nil }+func (s *lookupTestIndex) Audit(_ context.Context, actor, outcome string) error {+ s.outcomes = append(s.outcomes, actor+":"+outcome)+ return s.auditErr+}++type lookupTestDirectory struct{}++func (lookupTestDirectory) LookupAccount(_ context.Context, id string) (*keycloak.Account, error) {+ return &keycloak.Account{ID: id, Email: "[email protected]"}, nil+}++func TestAccountLookupStrictJSONAndDenialAudit(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ index := &lookupTestIndex{}+ s := accountlookup.New(ctx, index, lookupTestDirectory{}, "fixture", "fixture-key", nil)+ defer func() { cancel(); s.Wait() }()+ admin := true+ h := Handlers{AccountLookup: s, adminAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return admin, nil }}+ handler := PrivateAccountLookup(h.AuditAccountLookupAccess(http.HandlerFunc(h.LookupAccount)))+ for _, tc := range []struct {+ body string+ want int+ }{+ {`{"kind":"account_id","value":"fixture"}`, 200},+ {`{"kind":"account_id","value":"fixture","extra":"private-target"}`, 400},+ {`{"kind":"account_id","value":"fixture"} {}`, 400},+ {`{"kind":"account_id","value":"` + strings.Repeat("x", 1100) + `"}`, 400},+ } {+ r := httptest.NewRequest("POST", "/api/admin/account-lookup", strings.NewReader(tc.body))+ r = r.WithContext(context.WithValue(r.Context(), auth.UserKey, &auth.User{ID: "admin", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}))+ w := httptest.NewRecorder()+ handler.ServeHTTP(w, r)+ if w.Code != tc.want || w.Header().Get("Cache-Control") != "no-store" {+ t.Fatalf("response=%d %s", w.Code, w.Body.String())+ }+ if tc.want == 200 && !strings.Contains(w.Body.String(), "[email protected]") {+ t.Fatal("account not returned")+ }+ if tc.want != 200 && strings.Contains(w.Body.String(), "fixture") {+ t.Fatal("target leaked in error")+ }+ }+ admin = false+ r := httptest.NewRequest("POST", "/api/admin/account-lookup", strings.NewReader("private-target"))+ r = r.WithContext(context.WithValue(r.Context(), auth.UserKey, &auth.User{ID: "admin", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}))+ w := httptest.NewRecorder()+ handler.ServeHTTP(w, r)+ if w.Code != 403 || index.outcomes[len(index.outcomes)-1] != "admin:forbidden" {+ t.Fatal("denial not audited")+ }+ if strings.Contains(strings.Join(index.outcomes, " "), "fixture") {+ t.Fatal("target leaked into audit")+ }+}++func TestAccountLookupAuthorization(t *testing.T) {+ for _, tc := range []struct {+ name string+ user *auth.User+ admin bool+ err error+ want int+ }{+ {"anonymous", nil, false, nil, 403},+ {"customer", &auth.User{ID: "customer", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}, false, nil, 403},+ {"revoked", &auth.User{ID: "admin", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}, false, nil, 403},+ {"user key admin", &auth.User{ID: "admin", AZP: "ukey-test", PrincipalType: auth.PrincipalTypeUserKey}, true, nil, 403},+ {"CLI admin", &auth.User{ID: "admin", AZP: "cua-cli", PrincipalType: auth.PrincipalTypeUser}, true, nil, 403},+ {"automation spoof", &auth.User{ID: "admin", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeGitHubOIDC}, true, nil, 403},+ {"provider unavailable", &auth.User{ID: "admin", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}, false, errors.New("private error"), 503},+ {"no database", &auth.User{ID: "admin", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}, true, nil, 503},+ } {+ t.Run(tc.name, func(t *testing.T) {+ h := Handlers{adminAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return tc.admin, tc.err }}+ r := httptest.NewRequest("POST", "/api/admin/account-lookup", strings.NewReader(`{"kind":"account_id","value":"private-target"}`))+ if tc.user != nil {+ r = r.WithContext(context.WithValue(r.Context(), auth.UserKey, tc.user))+ }+ w := httptest.NewRecorder()+ h.LookupAccount(w, r)+ if w.Code != tc.want || w.Header().Get("Cache-Control") != "no-store" {+ t.Fatalf("status=%d headers=%v", w.Code, w.Header())+ }+ if strings.Contains(w.Body.String(), "private") {+ t.Fatal("sensitive data in error")+ }+ })+ }+}++func TestAccountLookupAuditFailureOverridesInvalidInput(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ index := &lookupTestIndex{auditErr: errors.New("private audit failure")}+ s := accountlookup.New(ctx, index, lookupTestDirectory{}, "fixture", "fixture-key", nil)+ defer func() { cancel(); s.Wait() }()+ h := Handlers{AccountLookup: s, adminAccessEvaluator: func(context.Context, *auth.User) (bool, error) { return true, nil }}+ r := httptest.NewRequest("POST", "/api/admin/account-lookup", strings.NewReader(`{}`))+ r = r.WithContext(context.WithValue(r.Context(), auth.UserKey, &auth.User{ID: "admin", AZP: "cyclops-cs-spa", PrincipalType: auth.PrincipalTypeUser}))+ w := httptest.NewRecorder()+ h.LookupAccount(w, r)+ if w.Code != http.StatusServiceUnavailable || strings.Contains(w.Body.String(), "private") {+ t.Fatalf("audit failure response=%d %s", w.Code, w.Body.String())+ }+}diff --git a/libs/fleet/backend/handlers/handlers.go b/libs/fleet/backend/handlers/handlers.goindex e4b04d15eb..0dd1fd9c8c 100644--- a/libs/fleet/backend/handlers/handlers.go+++ b/libs/fleet/backend/handlers/handlers.go@@ -12,6 +12,7 @@ import ("regexp""time"+ "cyclops-cs-backend/accountlookup""cyclops-cs-backend/auth""cyclops-cs-backend/chat""cyclops-cs-backend/config"@@ -37,6 +38,7 @@ type SignedServiceURLService interface {}type Handlers struct {+ AccountLookup *accountlookup.ServiceAdmin *keycloak.AdminGatewayCfg config.GatewayConfigurationAuthCfg config.AuthConfigurationdiff --git a/libs/fleet/backend/internal/redactederror/error.go b/libs/fleet/backend/internal/redactederror/error.gonew file mode 100644index 0000000000..9d7776d508--- /dev/null+++ b/libs/fleet/backend/internal/redactederror/error.go@@ -0,0 +1,18 @@+// Package redactederror preserves internal error causes without rendering them.+package redactederror++type redactedError struct {+ message string+ cause error+}++func (err *redactedError) Error() string { return err.message }+func (err *redactedError) Unwrap() error { return err.cause }++// New renders only message; callers must use a safe, static message.+func New(message string, cause error) error {+ if cause == nil {+ return nil+ }+ return &redactedError{message: message, cause: cause}+}diff --git a/libs/fleet/backend/internal/redactederror/error_test.go b/libs/fleet/backend/internal/redactederror/error_test.gonew file mode 100644index 0000000000..f55ab55860--- /dev/null+++ b/libs/fleet/backend/internal/redactederror/error_test.go@@ -0,0 +1,24 @@+package redactederror++import (+ "errors"+ "testing"+)++type privateError struct{}++func (*privateError) Error() string { return "private upstream detail" }++func TestNewPreservesCauseWithoutRenderingIt(t *testing.T) {+ private := &privateError{}+ sentinel := errors.New("public classification")+ cause := errors.Join(sentinel, private)+ err := New("safe message", cause)+ var typed *privateError+ if err.Error() != "safe message" || !errors.Is(err, sentinel) || !errors.Is(err, private) || !errors.As(err, &typed) || typed != private || errors.Unwrap(err) != cause {+ t.Fatal("redacted wrapper must preserve the complete cause chain")+ }+ if New("safe message", nil) != nil {+ t.Fatal("nil cause must remain nil")+ }+}diff --git a/libs/fleet/backend/keycloak/account_lookup.go b/libs/fleet/backend/keycloak/account_lookup.gonew file mode 100644index 0000000000..ef7ced7b4f--- /dev/null+++ b/libs/fleet/backend/keycloak/account_lookup.go@@ -0,0 +1,82 @@+package keycloak++import (+ "context"+ "errors"+ "net/http"+ "time"++ "cyclops-cs-backend/internal/redactederror"++ "github.com/Nerzal/gocloak/v13"+)++var ErrAccountDirectory = errors.New("account directory unavailable")++type Account struct {+ ID string `json:"id"`+ Username string `json:"username"`+ Email string `json:"email"`+ EmailVerified bool `json:"email_verified"`+ CreatedAt *time.Time `json:"created_at,omitempty"`+}++// Keep directory response bodies and identifiers out of errors and traces.+func (a *Admin) LookupAccount(ctx context.Context, subject string) (*Account, error) {+ token, err := a.accountLookupToken(ctx)+ if err != nil {+ return nil, redactederror.New(ErrAccountDirectory.Error(), errors.Join(ErrAccountDirectory, err))+ }+ u, err := a.client.GetUserByID(ctx, token, a.realm, subject)+ if err != nil {+ var apiErr *gocloak.APIError+ if errors.As(err, &apiErr) && apiErr.Code == http.StatusNotFound {+ return nil, nil+ }+ return nil, redactederror.New(ErrAccountDirectory.Error(), errors.Join(ErrAccountDirectory, err))+ }+ if u == nil || derefStr(u.ID) != subject {+ return nil, ErrAccountDirectory+ }+ result := &Account{ID: subject, Username: derefStr(u.Username), Email: derefStr(u.Email), EmailVerified: u.EmailVerified != nil && *u.EmailVerified}+ if u.CreatedTimestamp != nil {+ t := time.UnixMilli(*u.CreatedTimestamp).UTC()+ result.CreatedAt = &t+ }+ return result, nil+}++// ListAccountIDs is used only by the explicit offline backfill, never a lookup.+func (a *Admin) ListAccountIDs(ctx context.Context, first, max int) ([]string, error) {+ if first < 0 || max < 1 || max > 100 {+ return nil, ErrAccountDirectory+ }+ token, err := a.accountLookupToken(ctx)+ if err != nil {+ return nil, redactederror.New(ErrAccountDirectory.Error(), errors.Join(ErrAccountDirectory, err))+ }+ users, err := a.client.GetUsers(ctx, token, a.realm, gocloak.GetUsersParams{First: &first, Max: &max, BriefRepresentation: gocloak.BoolP(true)})+ if err != nil {+ return nil, redactederror.New(ErrAccountDirectory.Error(), errors.Join(ErrAccountDirectory, err))+ }+ ids := make([]string, 0, len(users))+ for _, u := range users {+ if u == nil || derefStr(u.ID) == "" {+ return nil, ErrAccountDirectory+ }+ ids = append(ids, *u.ID)+ }+ return ids, nil+}++func (a *Admin) accountLookupToken(ctx context.Context) (string, error) {+ // Unlike the general admin helper, do not record upstream error bodies.+ token, err := a.client.LoginClient(ctx, a.clientID, a.clientSecret, a.realm)+ if err != nil {+ return "", redactederror.New(ErrAccountDirectory.Error(), errors.Join(ErrAccountDirectory, err))+ }+ if token == nil || token.AccessToken == "" {+ return "", ErrAccountDirectory+ }+ return token.AccessToken, nil+}diff --git a/libs/fleet/backend/keycloak/account_lookup_test.go b/libs/fleet/backend/keycloak/account_lookup_test.gonew file mode 100644index 0000000000..7b74aedd90--- /dev/null+++ b/libs/fleet/backend/keycloak/account_lookup_test.go@@ -0,0 +1,56 @@+package keycloak++import (+ "context"+ "errors"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/Nerzal/gocloak/v13"+)++func TestAccountDirectoryExactAndSanitized(t *testing.T) {+ for _, code := range []int{200, 404, 403, 500} {+ t.Run(fmt.Sprint(code), func(t *testing.T) {+ calls := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if strings.HasSuffix(r.URL.Path, "/token") {+ fmt.Fprint(w, `{"access_token":"synthetic-token","expires_in":60}`)+ return+ }+ calls+++ if r.URL.Path != "/admin/realms/test/users/test-account" {+ t.Errorf("not exact path: %s", r.URL.Path)+ }+ w.WriteHeader(code)+ if code == 200 {+ fmt.Fprint(w, `{"id":"test-account","username":"example","email":"[email protected]","emailVerified":true,"createdTimestamp":1000}`)+ } else {+ fmt.Fprint(w, `{"errorMessage":"PRIVATE-DIRECTORY-DETAIL"}`)+ }+ }))+ defer server.Close()+ a := NewAdmin(server.URL, "test", "test-client", "synthetic-secret", "key-", "ukey-")+ account, err := a.LookupAccount(context.Background(), "test-account")+ if code == 200 && (err != nil || account == nil || account.Username != "example" || account.CreatedAt == nil) {+ t.Fatalf("account=%+v err=%v", account, err)+ }+ if code == 404 && (err != nil || account != nil) {+ t.Fatal("404 must be absent account")+ }+ if code >= 400 && code != 404 {+ var apiErr *gocloak.APIError+ if !errors.Is(err, ErrAccountDirectory) || err.Error() != ErrAccountDirectory.Error() || !errors.As(err, &apiErr) || apiErr.Code != code || errors.Unwrap(err) == nil {+ t.Fatal("directory failure must redact text and retain the API error")+ }+ }+ if calls != 1 {+ t.Fatal("not a single exact lookup")+ }+ })+ }+}diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.goindex a53783976b..0ddbcd756a 100644--- a/libs/fleet/backend/main.go+++ b/libs/fleet/backend/main.go@@ -35,6 +35,7 @@ import ("github.com/spf13/cobra"+ "cyclops-cs-backend/accountlookup""cyclops-cs-backend/auth""cyclops-cs-backend/billing""cyclops-cs-backend/chat"@@ -127,6 +128,18 @@ func onlyLog(route string, h http.HandlerFunc) http.Handler {}func setupRouter(c handlers.Handlers) http.Handler {+ // Observe validated, owner-normalized identities without blocking requests.+ authenticated := withAuthenticatedMiddlewares+ withAuthenticatedMiddlewares := func(route string, handler http.HandlerFunc, observers ...auth.Middleware) http.Handler {+ if c.AccountLookup != nil {+ observers = append([]auth.Middleware{c.AccountLookup.Observe}, observers...)+ }+ wrapped := authenticated(route, handler, observers...)+ if route == "/api/admin/account-lookup" {+ return handlers.PrivateAccountLookup(wrapped)+ }+ return wrapped+ }// The namespace-ownership conjunct on /api/svc and// /api/namespaces/{name} asks Kubernetes a question, through a probe that// is a handlers method. auth cannot import handlers, so the policy names@@ -172,6 +185,7 @@ func setupRouter(c handlers.Handlers) http.Handler {r.Handle("GET /api/usage/pool", withAuthenticatedMiddlewares("/api/usage/pool", c.GetUsagePoolDetail))r.Handle("POST /api/usage/browser-timings", withAuthenticatedMiddlewares("/api/usage/browser-timings", c.RecordUsageBrowserTimings))r.Handle("GET /api/admin/feature-flags", withAuthenticatedMiddlewares("/api/admin/feature-flags", c.ListFeatureFlags))+ r.Handle("POST /api/admin/account-lookup", withAuthenticatedMiddlewares("/api/admin/account-lookup", c.LookupAccount, c.AuditAccountLookupAccess))r.Handle("POST /api/admin/feature-flags", withAuthenticatedMiddlewares("/api/admin/feature-flags", c.CreateFeatureFlag))r.Handle("PUT /api/admin/feature-flags/{key}", withAuthenticatedMiddlewares("/api/admin/feature-flags/{key}", c.UpdateFeatureFlag))r.Handle("DELETE /api/admin/feature-flags/{key}", withAuthenticatedMiddlewares("/api/admin/feature-flags/{key}", c.DeleteFeatureFlag))@@ -400,6 +414,9 @@ func run() error {h := handlers.New(admin, cfg)h.Analytics = analyticsClient+ lookupService, closeLookup, lookupErr := initializeAccountLookup(ctx, cfg, admin)+ h.AccountLookup = lookupService+ defer closeLookup()signedServiceURLsContext, cancelSignedServiceURLs := context.WithCancel(ctx)defer cancelSignedServiceURLs()var signedServiceURLsRetry sync.WaitGroup@@ -421,7 +438,7 @@ func run() error {defer shutdownSignedServiceURLs(cancelSignedServiceURLs, &signedServiceURLsRetry, &signedServiceURLsMu, &closeSignedServiceURLs)usageProvider, closeUsageProvider, err := initializeUsageProvider(ctx, cfg.Usage)if err != nil {- return errors.Join(fmt.Errorf("initialize usage provider: %w", err), startupErrors, telemetryErr)+ return errors.Join(fmt.Errorf("initialize usage provider: %w", err), startupErrors, telemetryErr, lookupErr)}defer closeUsageProvider()h.Usage = usageProvider@@ -435,7 +452,7 @@ func run() error {if cfg.Database.URL != "" {conversationStore, conversationStoreErr := chat.NewPostgresConversationStore(ctx, cfg.Database.URL)if conversationStoreErr != nil {- return errors.Join(fmt.Errorf("initialize chat conversation store: %w", conversationStoreErr), startupErrors, telemetryErr)+ return errors.Join(fmt.Errorf("initialize chat conversation store: %w", conversationStoreErr), startupErrors, telemetryErr, lookupErr)}defer conversationStore.Close()h.Conversations = conversationStore@@ -484,7 +501,22 @@ func run() error {router := setupRouter(h)srv := &http.Server{Addr: cfg.WebServer.Addr, Handler: router}- return errors.Join(serveUntilCanceled(ctx, srv), startupErrors, telemetryErr)+ return errors.Join(serveUntilCanceled(ctx, srv), startupErrors, telemetryErr, lookupErr)+}++// Lazy pool connections keep this optional feature independent of serving readiness.+func initializeAccountLookup(ctx context.Context, cfg *config.Configuration, admin *keycloak.Admin) (*accountlookup.Service, func(), error) {+ if cfg.Database.URL == "" || cfg.ProductAnalytics.IdentityKey == "" {+ return nil, func() {}, nil+ }+ store, err := accountlookup.NewStore(ctx, cfg.Database.URL)+ if err != nil {+ slog.Warn("account lookup unavailable: invalid database configuration")+ return nil, func() {}, err+ }+ lookupCtx, cancel := context.WithCancel(ctx)+ service := accountlookup.New(lookupCtx, store, admin, cfg.Keycloak.Realm, cfg.ProductAnalytics.IdentityKey, cfg.ProductAnalytics.ExcludedSubjects)+ return service, func() { cancel(); service.Wait(); store.Close() }, nil}// startSignedServiceURLs installs the signed service URL dependency on firstdiff --git a/libs/fleet/backend/main_test.go b/libs/fleet/backend/main_test.goindex 161e700110..eec1a7e213 100644--- a/libs/fleet/backend/main_test.go+++ b/libs/fleet/backend/main_test.go@@ -45,6 +45,35 @@ import (type auditWiringLock struct{}+func TestInitializeAccountLookupOptionalAndLazy(t *testing.T) {+ for _, test := range []struct {+ name, databaseURL, identityKey string+ wantService, wantError bool+ }{+ {name: "disabled without database", identityKey: "test-key"},+ {name: "disabled without identity key", databaseURL: "invalid"},+ {name: "invalid configuration", databaseURL: "postgres://sensitive:value@localhost/db?port=invalid", identityKey: "test-key", wantError: true},+ {name: "unreachable database remains lazy", databaseURL: "postgres://test:[email protected]:1/test?connect_timeout=1", identityKey: "test-key", wantService: true},+ } {+ t.Run(test.name, func(t *testing.T) {+ cfg := &config.Configuration{}+ cfg.Database.URL = test.databaseURL+ cfg.ProductAnalytics.IdentityKey = test.identityKey+ service, closeLookup, err := initializeAccountLookup(context.Background(), cfg, nil)+ if closeLookup == nil {+ t.Fatal("missing cleanup function")+ }+ defer closeLookup()+ if (service != nil) != test.wantService || (err != nil) != test.wantError {+ t.Fatalf("service present=%v, error=%v", service != nil, err)+ }+ if err != nil && (errors.Unwrap(err) == nil || strings.Contains(err.Error(), "sensitive")) {+ t.Fatal("initialization error must preserve its cause without rendering credentials")+ }+ })+ }+}+func (auditWiringLock) WithLock(ctx context.Context, callback func(context.Context) error) error {return callback(ctx)}