Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(iam): move group caches to store with LRU+TTL (#18870)
p0nybc6768a8f92a
diff --git a/backend/api/v1/auth_service.go b/backend/api/v1/auth_service.goindex 8c4dc8fec1..54d261ae4e 100644--- a/backend/api/v1/auth_service.go+++ b/backend/api/v1/auth_service.go@@ -785,7 +785,11 @@ func (s *AuthService) finalizeLogin(ctx context.Context, req *connect.Request[v1slog.Error("failed to update user profile", log.BBError(err), slog.String("user", user.Email))}- response.User = convertToUser(ctx, s.iamManager, user)+ v1User, err := convertToUser(ctx, s.iamManager, user)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))+ }+ response.User = v1Userreturn resp, nil}diff --git a/backend/api/v1/user_service.go b/backend/api/v1/user_service.goindex 2c30bd096f..4f209a538a 100644--- a/backend/api/v1/user_service.go+++ b/backend/api/v1/user_service.go@@ -63,7 +63,11 @@ func (s *UserService) GetUser(ctx context.Context, request *connect.Request[v1pbif user == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("user %q not found", email))}- return connect.NewResponse(convertToUser(ctx, s.iamManager, user)), nil+ v1User, err := convertToUser(ctx, s.iamManager, user)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))+ }+ return connect.NewResponse(v1User), nil}// BatchGetUsers get users in batch.@@ -85,7 +89,11 @@ func (s *UserService) GetCurrentUser(ctx context.Context, _ *connect.Request[empif !ok || user == nil {return nil, connect.NewError(connect.CodeUnauthenticated, errors.Errorf("authenticated user not found"))}- return connect.NewResponse(convertToUser(ctx, s.iamManager, user)), nil+ v1User, err := convertToUser(ctx, s.iamManager, user)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))+ }+ return connect.NewResponse(v1User), nil}// ListUsers lists all users.@@ -144,7 +152,11 @@ func (s *UserService) ListUsers(ctx context.Context, request *connect.Request[v1NextPageToken: nextPageToken,}for _, user := range users {- response.Users = append(response.Users, convertToUser(ctx, s.iamManager, user))+ v1User, err := convertToUser(ctx, s.iamManager, user)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))+ }+ response.Users = append(response.Users, v1User)}return connect.NewResponse(response), nil}@@ -303,7 +315,10 @@ func (s *UserService) CreateUser(ctx context.Context, request *connect.Request[v}}- userResponse := convertToUser(ctx, s.iamManager, user)+ userResponse, err := convertToUser(ctx, s.iamManager, user)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))+ }if request.Msg.User.UserType == v1pb.UserType_SERVICE_ACCOUNT {userResponse.ServiceKey = password}@@ -526,7 +541,10 @@ func (s *UserService) UpdateUser(ctx context.Context, request *connect.Request[vreturn nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to update user"))}- userResponse := convertToUser(ctx, s.iamManager, user)+ userResponse, err := convertToUser(ctx, s.iamManager, user)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))+ }if request.Msg.User.UserType == v1pb.UserType_SERVICE_ACCOUNT && passwordPatch != nil {userResponse.ServiceKey = *passwordPatch}@@ -664,7 +682,11 @@ func (s *UserService) UndeleteUser(ctx context.Context, request *connect.Requestif err != nil {return nil, connect.NewError(connect.CodeInternal, err)}- return connect.NewResponse(convertToUser(ctx, s.iamManager, user)), nil+ v1User, err := convertToUser(ctx, s.iamManager, user)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))+ }+ return connect.NewResponse(v1User), nil}// UpdateEmail updates a user's email address.@@ -722,7 +744,11 @@ func (s *UserService) UpdateEmail(ctx context.Context, request *connect.Request[return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to update user email"))}- return connect.NewResponse(convertToUser(ctx, s.iamManager, user)), nil+ v1User, err := convertToUser(ctx, s.iamManager, user)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))+ }+ return connect.NewResponse(v1User), nil}func convertToV1UserType(userType storepb.PrincipalType) v1pb.UserType {@@ -740,7 +766,11 @@ func convertToV1UserType(userType storepb.PrincipalType) v1pb.UserType {}}-func convertToUser(ctx context.Context, iamManager *iam.Manager, user *store.UserMessage) *v1pb.User {+func convertToUser(ctx context.Context, iamManager *iam.Manager, user *store.UserMessage) (*v1pb.User, error) {+ groups, err := iamManager.GetUserGroups(ctx, user.Email)+ if err != nil {+ return nil, err+ }convertedUser := &v1pb.User{Name: common.FormatUserEmail(user.Email),State: convertDeletedToState(user.MemberDeleted),@@ -753,7 +783,7 @@ func convertToUser(ctx context.Context, iamManager *iam.Manager, user *store.UseLastChangePasswordTime: user.Profile.LastChangePasswordTime,Source: user.Profile.Source,},- Groups: iamManager.GetUserGroups(user.Email),+ Groups: groups,}// Add workload identity config if present@@ -776,7 +806,7 @@ func convertToUser(ctx context.Context, iamManager *iam.Manager, user *store.UseconvertedUser.TempOtpSecretCreatedTime = user.MFAConfig.TempOtpSecretCreatedTime}}- return convertedUser+ return convertedUser, nil}func convertToPrincipalType(userType v1pb.UserType) (storepb.PrincipalType, error) {diff --git a/backend/component/iam/manager.go b/backend/component/iam/manager.goindex fd25915277..a1ebf0cdf5 100644--- a/backend/component/iam/manager.go+++ b/backend/component/iam/manager.go@@ -12,13 +12,9 @@ import ("github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store"- "github.com/bytebase/bytebase/backend/utils")type Manager struct {- groupMembers map[string]map[string]bool- // member - groups mapping- memberGroups map[string][]stringstore *store.StorelicenseService *enterprise.LicenseService}@@ -39,12 +35,16 @@ func (m *Manager) CheckPermission(ctx context.Context, p permission.Permission,perms, _ := m.GetPermissions(ctx, role)return perms}+ getGroupMembers := func(groupName string) map[string]bool {+ members, _ := m.store.GetGroupMembersSnapshot(ctx, groupName)+ return members+ }policyMessage, err := m.store.GetWorkspaceIamPolicy(ctx)if err != nil {return false, err}- if ok := check(user, p, policyMessage.Policy, getPermissions, m.groupMembers); ok {+ if ok := check(user, p, policyMessage.Policy, getPermissions, getGroupMembers); ok {return true, nil}@@ -65,7 +65,7 @@ func (m *Manager) CheckPermission(ctx context.Context, p permission.Permission,if err != nil {return false, err}- if ok := check(user, p, policyMessage.Policy, getPermissions, m.groupMembers); !ok {+ if ok := check(user, p, policyMessage.Policy, getPermissions, getGroupMembers); !ok {allOK = falsebreak}@@ -75,27 +75,8 @@ func (m *Manager) CheckPermission(ctx context.Context, p permission.Permission,return false, nil}-func (m *Manager) ReloadCache(ctx context.Context) error {- groups, err := m.store.ListGroups(ctx, &store.FindGroupMessage{})- if err != nil {- return err- }- groupMembers := make(map[string]map[string]bool)- memberGroups := make(map[string][]string)- for _, group := range groups {- usersSet := make(map[string]bool)- groupName := utils.FormatGroupName(group)- for _, m := range group.Payload.GetMembers() {- usersSet[m.Member] = true- if _, ok := memberGroups[m.Member]; !ok {- memberGroups[m.Member] = []string{}- }- memberGroups[m.Member] = append(memberGroups[m.Member], groupName)- }- groupMembers[groupName] = usersSet- }- m.groupMembers = groupMembers- m.memberGroups = memberGroups+func (m *Manager) ReloadCache(_ context.Context) error {+ m.store.PurgeGroupCaches()return nil}@@ -113,11 +94,11 @@ func (m *Manager) GetPermissions(ctx context.Context, roleName string) (map[permreturn maps.Clone(role.Permissions), nil}-func (m *Manager) GetUserGroups(email string) []string {- return m.memberGroups[common.FormatUserEmail(email)]+func (m *Manager) GetUserGroups(ctx context.Context, email string) ([]string, error) {+ return m.store.GetUserGroupsSnapshot(ctx, common.FormatUserEmail(email))}-func check(user *store.UserMessage, p permission.Permission, policy *storepb.IamPolicy, getPermissions func(role string) map[permission.Permission]bool, groupMembers map[string]map[string]bool) bool {+func check(user *store.UserMessage, p permission.Permission, policy *storepb.IamPolicy, getPermissions func(role string) map[permission.Permission]bool, getGroupMembers func(groupName string) map[string]bool) bool {userName := common.FormatUserEmail(user.Email)for _, binding := range policy.GetBindings() {@@ -136,8 +117,8 @@ func check(user *store.UserMessage, p permission.Permission, policy *storepb.Iamreturn true}if strings.HasPrefix(member, common.GroupPrefix) {- if groupMembers, ok := groupMembers[member]; ok {- if groupMembers[userName] {+ if members := getGroupMembers(member); members != nil {+ if members[userName] {return true}}diff --git a/backend/component/iam/manager_test.go b/backend/component/iam/manager_test.goindex e023d5c3c9..4844d968fb 100644--- a/backend/component/iam/manager_test.go+++ b/backend/component/iam/manager_test.go@@ -102,7 +102,13 @@ func TestCheck(t *testing.T) {}}for i, test := range tests {- got := check(testUser, test.permission, test.policy, getPermissions, test.groupMembers)+ getGroupMembers := func(groupName string) map[string]bool {+ if test.groupMembers == nil {+ return nil+ }+ return test.groupMembers[groupName]+ }+ got := check(testUser, test.permission, test.policy, getPermissions, getGroupMembers)if got != test.want {require.Equal(t, test.want, got, i)}diff --git a/backend/store/group.go b/backend/store/group.goindex ffe2e62015..a964b9fb12 100644--- a/backend/store/group.go+++ b/backend/store/group.go@@ -311,6 +311,7 @@ func (s *Store) UpdateGroup(ctx context.Context, patch *UpdateGroupMessage) (*Grif group.Email != "" {s.groupCache.Add(group.Email, &group)+ s.groupMembersCache.Remove("groups/" + group.Email)}return &group, nil}@@ -340,10 +341,67 @@ func (s *Store) DeleteGroup(ctx context.Context, id string) error {if email.Valid && email.String != "" {s.groupCache.Remove(email.String)+ s.groupMembersCache.Remove("groups/" + email.String)}return nil}+// GetUserGroupsSnapshot returns groups for a user with snapshot reads (with cache).+// userName format is "users/{email}".+// Trades consistency for performance.+func (s *Store) GetUserGroupsSnapshot(ctx context.Context, userName string) ([]string, error) {+ if v, ok := s.memberGroupsCache.Get(userName); ok {+ return v, nil+ }++ groups, err := s.ListGroups(ctx, &FindGroupMessage{})+ if err != nil {+ return nil, err+ }++ var userGroups []string+ for _, group := range groups {+ for _, m := range group.Payload.GetMembers() {+ if m.Member == userName {+ groupName := common.FormatGroupEmail(group.Email)+ if group.Email == "" {+ groupName = common.FormatGroupEmail(group.ID)+ }+ userGroups = append(userGroups, groupName)+ break+ }+ }+ }+ s.memberGroupsCache.Add(userName, userGroups)+ return userGroups, nil+}++// GetGroupMembersSnapshot returns group members with snapshot reads (with cache).+// groupName format is "groups/{email}".+// Trades consistency for performance.+func (s *Store) GetGroupMembersSnapshot(ctx context.Context, groupName string) (map[string]bool, error) {+ if v, ok := s.groupMembersCache.Get(groupName); ok {+ return v, nil+ }++ // Extract email from group name "groups/{email}"+ email := strings.TrimPrefix(groupName, "groups/")+ group, err := s.GetGroup(ctx, &FindGroupMessage{Email: &email})+ if err != nil {+ return nil, err+ }+ if group == nil {+ return nil, nil+ }++ members := make(map[string]bool)+ for _, m := range group.Payload.GetMembers() {+ members[m.Member] = true+ }+ s.groupMembersCache.Add(groupName, members)+ return members, nil+}+func GetListGroupFilter(find *FindGroupMessage, filter string) (*qb.Query, error) {if filter == "" {return nil, nildiff --git a/backend/store/store.go b/backend/store/store.goindex b9c6bf9cfe..a7d73dfc34 100644--- a/backend/store/store.go+++ b/backend/store/store.go@@ -19,15 +19,17 @@ type Store struct {enableCache bool// Cache.- Secret string- userEmailCache *lru.Cache[string, *UserMessage]- instanceCache *lru.Cache[string, *InstanceMessage]- databaseCache *lru.Cache[string, *DatabaseMessage]- projectCache *lru.Cache[string, *ProjectMessage]- policyCache *lru.Cache[string, *PolicyMessage]- settingCache *lru.Cache[storepb.SettingName, *SettingMessage]- rolesCache *expirable.LRU[string, *RoleMessage]- groupCache *lru.Cache[string, *GroupMessage]+ Secret string+ userEmailCache *lru.Cache[string, *UserMessage]+ instanceCache *lru.Cache[string, *InstanceMessage]+ databaseCache *lru.Cache[string, *DatabaseMessage]+ projectCache *lru.Cache[string, *ProjectMessage]+ policyCache *lru.Cache[string, *PolicyMessage]+ settingCache *lru.Cache[storepb.SettingName, *SettingMessage]+ rolesCache *expirable.LRU[string, *RoleMessage]+ groupCache *expirable.LRU[string, *GroupMessage]+ groupMembersCache *expirable.LRU[string, map[string]bool]+ memberGroupsCache *expirable.LRU[string, []string]// Large objects.sheetFullCache *lru.Cache[string, *SheetMessage]@@ -65,10 +67,9 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {if err != nil {return nil, err}- groupCache, err := lru.New[string, *GroupMessage](1024)- if err != nil {- return nil, err- }+ groupCache := expirable.NewLRU[string, *GroupMessage](1024, nil, time.Minute)+ groupMembersCache := expirable.NewLRU[string, map[string]bool](1024, nil, time.Minute)+ memberGroupsCache := expirable.NewLRU[string, []string](4096, nil, time.Minute)// Initialize database connection (handles both direct URL and file-based)dbConnManager := NewDBConnectionManager(pgURL)@@ -81,15 +82,17 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {enableCache: enableCache,// Cache.- userEmailCache: userEmailCache,- instanceCache: instanceCache,- databaseCache: databaseCache,- projectCache: projectCache,- policyCache: policyCache,- settingCache: settingCache,- rolesCache: rolesCache,- sheetFullCache: sheetFullCache,- groupCache: groupCache,+ userEmailCache: userEmailCache,+ instanceCache: instanceCache,+ databaseCache: databaseCache,+ projectCache: projectCache,+ policyCache: policyCache,+ settingCache: settingCache,+ rolesCache: rolesCache,+ sheetFullCache: sheetFullCache,+ groupCache: groupCache,+ groupMembersCache: groupMembersCache,+ memberGroupsCache: memberGroupsCache,}return s, nil@@ -111,6 +114,13 @@ func (s *Store) DeleteCache() {s.userEmailCache.Purge()}+// PurgeGroupCaches purges all group-related caches.+func (s *Store) PurgeGroupCaches() {+ s.groupCache.Purge()+ s.groupMembersCache.Purge()+ s.memberGroupsCache.Purge()+}+func getInstanceCacheKey(instanceID string) string {return instanceID}