Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(iam): move group caches to store with LRU+TTL (#18870)

p0nybc6768a8f92a
diff --git a/backend/api/v1/auth_service.go b/backend/api/v1/auth_service.go
index 8c4dc8fec1..54d261ae4e 100644
--- a/backend/api/v1/auth_service.go
+++ b/backend/api/v1/auth_service.go
@@ -785,7 +785,11 @@ func (s *AuthService) finalizeLogin(ctx context.Context, req *connect.Request[v1
slog.Error("failed to update user profile", log.BBError(err), slog.String("user", user.Email))
}
- response.User = convertToUser(ctx, s.iamManager, user)
+ v1User, err := convertToUser(ctx, s.iamManager, user)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))
+ }
+ response.User = v1User
return resp, nil
}
diff --git a/backend/api/v1/user_service.go b/backend/api/v1/user_service.go
index 2c30bd096f..4f209a538a 100644
--- a/backend/api/v1/user_service.go
+++ b/backend/api/v1/user_service.go
@@ -63,7 +63,11 @@ func (s *UserService) GetUser(ctx context.Context, request *connect.Request[v1pb
if user == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("user %q not found", email))
}
- return connect.NewResponse(convertToUser(ctx, s.iamManager, user)), nil
+ v1User, err := convertToUser(ctx, s.iamManager, user)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))
+ }
+ return connect.NewResponse(v1User), nil
}
// BatchGetUsers get users in batch.
@@ -85,7 +89,11 @@ func (s *UserService) GetCurrentUser(ctx context.Context, _ *connect.Request[emp
if !ok || user == nil {
return nil, connect.NewError(connect.CodeUnauthenticated, errors.Errorf("authenticated user not found"))
}
- return connect.NewResponse(convertToUser(ctx, s.iamManager, user)), nil
+ v1User, err := convertToUser(ctx, s.iamManager, user)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))
+ }
+ return connect.NewResponse(v1User), nil
}
// ListUsers lists all users.
@@ -144,7 +152,11 @@ func (s *UserService) ListUsers(ctx context.Context, request *connect.Request[v1
NextPageToken: nextPageToken,
}
for _, user := range users {
- response.Users = append(response.Users, convertToUser(ctx, s.iamManager, user))
+ v1User, err := convertToUser(ctx, s.iamManager, user)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))
+ }
+ response.Users = append(response.Users, v1User)
}
return connect.NewResponse(response), nil
}
@@ -303,7 +315,10 @@ func (s *UserService) CreateUser(ctx context.Context, request *connect.Request[v
}
}
- userResponse := convertToUser(ctx, s.iamManager, user)
+ userResponse, err := convertToUser(ctx, s.iamManager, user)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))
+ }
if request.Msg.User.UserType == v1pb.UserType_SERVICE_ACCOUNT {
userResponse.ServiceKey = password
}
@@ -526,7 +541,10 @@ func (s *UserService) UpdateUser(ctx context.Context, request *connect.Request[v
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to update user"))
}
- userResponse := convertToUser(ctx, s.iamManager, user)
+ userResponse, err := convertToUser(ctx, s.iamManager, user)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))
+ }
if request.Msg.User.UserType == v1pb.UserType_SERVICE_ACCOUNT && passwordPatch != nil {
userResponse.ServiceKey = *passwordPatch
}
@@ -664,7 +682,11 @@ func (s *UserService) UndeleteUser(ctx context.Context, request *connect.Request
if err != nil {
return nil, connect.NewError(connect.CodeInternal, err)
}
- return connect.NewResponse(convertToUser(ctx, s.iamManager, user)), nil
+ v1User, err := convertToUser(ctx, s.iamManager, user)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))
+ }
+ return connect.NewResponse(v1User), nil
}
// UpdateEmail updates a user's email address.
@@ -722,7 +744,11 @@ func (s *UserService) UpdateEmail(ctx context.Context, request *connect.Request[
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to update user email"))
}
- return connect.NewResponse(convertToUser(ctx, s.iamManager, user)), nil
+ v1User, err := convertToUser(ctx, s.iamManager, user)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert user"))
+ }
+ return connect.NewResponse(v1User), nil
}
func convertToV1UserType(userType storepb.PrincipalType) v1pb.UserType {
@@ -740,7 +766,11 @@ func convertToV1UserType(userType storepb.PrincipalType) v1pb.UserType {
}
}
-func convertToUser(ctx context.Context, iamManager *iam.Manager, user *store.UserMessage) *v1pb.User {
+func convertToUser(ctx context.Context, iamManager *iam.Manager, user *store.UserMessage) (*v1pb.User, error) {
+ groups, err := iamManager.GetUserGroups(ctx, user.Email)
+ if err != nil {
+ return nil, err
+ }
convertedUser := &v1pb.User{
Name: common.FormatUserEmail(user.Email),
State: convertDeletedToState(user.MemberDeleted),
@@ -753,7 +783,7 @@ func convertToUser(ctx context.Context, iamManager *iam.Manager, user *store.Use
LastChangePasswordTime: user.Profile.LastChangePasswordTime,
Source: user.Profile.Source,
},
- Groups: iamManager.GetUserGroups(user.Email),
+ Groups: groups,
}
// Add workload identity config if present
@@ -776,7 +806,7 @@ func convertToUser(ctx context.Context, iamManager *iam.Manager, user *store.Use
convertedUser.TempOtpSecretCreatedTime = user.MFAConfig.TempOtpSecretCreatedTime
}
}
- return convertedUser
+ return convertedUser, nil
}
func convertToPrincipalType(userType v1pb.UserType) (storepb.PrincipalType, error) {
diff --git a/backend/component/iam/manager.go b/backend/component/iam/manager.go
index fd25915277..a1ebf0cdf5 100644
--- a/backend/component/iam/manager.go
+++ b/backend/component/iam/manager.go
@@ -12,13 +12,9 @@ import (
"github.com/bytebase/bytebase/backend/enterprise"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
- "github.com/bytebase/bytebase/backend/utils"
)
type Manager struct {
- groupMembers map[string]map[string]bool
- // member - groups mapping
- memberGroups map[string][]string
store *store.Store
licenseService *enterprise.LicenseService
}
@@ -39,12 +35,16 @@ func (m *Manager) CheckPermission(ctx context.Context, p permission.Permission,
perms, _ := m.GetPermissions(ctx, role)
return perms
}
+ getGroupMembers := func(groupName string) map[string]bool {
+ members, _ := m.store.GetGroupMembersSnapshot(ctx, groupName)
+ return members
+ }
policyMessage, err := m.store.GetWorkspaceIamPolicy(ctx)
if err != nil {
return false, err
}
- if ok := check(user, p, policyMessage.Policy, getPermissions, m.groupMembers); ok {
+ if ok := check(user, p, policyMessage.Policy, getPermissions, getGroupMembers); ok {
return true, nil
}
@@ -65,7 +65,7 @@ func (m *Manager) CheckPermission(ctx context.Context, p permission.Permission,
if err != nil {
return false, err
}
- if ok := check(user, p, policyMessage.Policy, getPermissions, m.groupMembers); !ok {
+ if ok := check(user, p, policyMessage.Policy, getPermissions, getGroupMembers); !ok {
allOK = false
break
}
@@ -75,27 +75,8 @@ func (m *Manager) CheckPermission(ctx context.Context, p permission.Permission,
return false, nil
}
-func (m *Manager) ReloadCache(ctx context.Context) error {
- groups, err := m.store.ListGroups(ctx, &store.FindGroupMessage{})
- if err != nil {
- return err
- }
- groupMembers := make(map[string]map[string]bool)
- memberGroups := make(map[string][]string)
- for _, group := range groups {
- usersSet := make(map[string]bool)
- groupName := utils.FormatGroupName(group)
- for _, m := range group.Payload.GetMembers() {
- usersSet[m.Member] = true
- if _, ok := memberGroups[m.Member]; !ok {
- memberGroups[m.Member] = []string{}
- }
- memberGroups[m.Member] = append(memberGroups[m.Member], groupName)
- }
- groupMembers[groupName] = usersSet
- }
- m.groupMembers = groupMembers
- m.memberGroups = memberGroups
+func (m *Manager) ReloadCache(_ context.Context) error {
+ m.store.PurgeGroupCaches()
return nil
}
@@ -113,11 +94,11 @@ func (m *Manager) GetPermissions(ctx context.Context, roleName string) (map[perm
return maps.Clone(role.Permissions), nil
}
-func (m *Manager) GetUserGroups(email string) []string {
- return m.memberGroups[common.FormatUserEmail(email)]
+func (m *Manager) GetUserGroups(ctx context.Context, email string) ([]string, error) {
+ return m.store.GetUserGroupsSnapshot(ctx, common.FormatUserEmail(email))
}
-func check(user *store.UserMessage, p permission.Permission, policy *storepb.IamPolicy, getPermissions func(role string) map[permission.Permission]bool, groupMembers map[string]map[string]bool) bool {
+func check(user *store.UserMessage, p permission.Permission, policy *storepb.IamPolicy, getPermissions func(role string) map[permission.Permission]bool, getGroupMembers func(groupName string) map[string]bool) bool {
userName := common.FormatUserEmail(user.Email)
for _, binding := range policy.GetBindings() {
@@ -136,8 +117,8 @@ func check(user *store.UserMessage, p permission.Permission, policy *storepb.Iam
return true
}
if strings.HasPrefix(member, common.GroupPrefix) {
- if groupMembers, ok := groupMembers[member]; ok {
- if groupMembers[userName] {
+ if members := getGroupMembers(member); members != nil {
+ if members[userName] {
return true
}
}
diff --git a/backend/component/iam/manager_test.go b/backend/component/iam/manager_test.go
index e023d5c3c9..4844d968fb 100644
--- a/backend/component/iam/manager_test.go
+++ b/backend/component/iam/manager_test.go
@@ -102,7 +102,13 @@ func TestCheck(t *testing.T) {
}}
for i, test := range tests {
- got := check(testUser, test.permission, test.policy, getPermissions, test.groupMembers)
+ getGroupMembers := func(groupName string) map[string]bool {
+ if test.groupMembers == nil {
+ return nil
+ }
+ return test.groupMembers[groupName]
+ }
+ got := check(testUser, test.permission, test.policy, getPermissions, getGroupMembers)
if got != test.want {
require.Equal(t, test.want, got, i)
}
diff --git a/backend/store/group.go b/backend/store/group.go
index ffe2e62015..a964b9fb12 100644
--- a/backend/store/group.go
+++ b/backend/store/group.go
@@ -311,6 +311,7 @@ func (s *Store) UpdateGroup(ctx context.Context, patch *UpdateGroupMessage) (*Gr
if group.Email != "" {
s.groupCache.Add(group.Email, &group)
+ s.groupMembersCache.Remove("groups/" + group.Email)
}
return &group, nil
}
@@ -340,10 +341,67 @@ func (s *Store) DeleteGroup(ctx context.Context, id string) error {
if email.Valid && email.String != "" {
s.groupCache.Remove(email.String)
+ s.groupMembersCache.Remove("groups/" + email.String)
}
return nil
}
+// GetUserGroupsSnapshot returns groups for a user with snapshot reads (with cache).
+// userName format is "users/{email}".
+// Trades consistency for performance.
+func (s *Store) GetUserGroupsSnapshot(ctx context.Context, userName string) ([]string, error) {
+ if v, ok := s.memberGroupsCache.Get(userName); ok {
+ return v, nil
+ }
+
+ groups, err := s.ListGroups(ctx, &FindGroupMessage{})
+ if err != nil {
+ return nil, err
+ }
+
+ var userGroups []string
+ for _, group := range groups {
+ for _, m := range group.Payload.GetMembers() {
+ if m.Member == userName {
+ groupName := common.FormatGroupEmail(group.Email)
+ if group.Email == "" {
+ groupName = common.FormatGroupEmail(group.ID)
+ }
+ userGroups = append(userGroups, groupName)
+ break
+ }
+ }
+ }
+ s.memberGroupsCache.Add(userName, userGroups)
+ return userGroups, nil
+}
+
+// GetGroupMembersSnapshot returns group members with snapshot reads (with cache).
+// groupName format is "groups/{email}".
+// Trades consistency for performance.
+func (s *Store) GetGroupMembersSnapshot(ctx context.Context, groupName string) (map[string]bool, error) {
+ if v, ok := s.groupMembersCache.Get(groupName); ok {
+ return v, nil
+ }
+
+ // Extract email from group name "groups/{email}"
+ email := strings.TrimPrefix(groupName, "groups/")
+ group, err := s.GetGroup(ctx, &FindGroupMessage{Email: &email})
+ if err != nil {
+ return nil, err
+ }
+ if group == nil {
+ return nil, nil
+ }
+
+ members := make(map[string]bool)
+ for _, m := range group.Payload.GetMembers() {
+ members[m.Member] = true
+ }
+ s.groupMembersCache.Add(groupName, members)
+ return members, nil
+}
+
func GetListGroupFilter(find *FindGroupMessage, filter string) (*qb.Query, error) {
if filter == "" {
return nil, nil
diff --git a/backend/store/store.go b/backend/store/store.go
index b9c6bf9cfe..a7d73dfc34 100644
--- a/backend/store/store.go
+++ b/backend/store/store.go
@@ -19,15 +19,17 @@ type Store struct {
enableCache bool
// Cache.
- Secret string
- userEmailCache *lru.Cache[string, *UserMessage]
- instanceCache *lru.Cache[string, *InstanceMessage]
- databaseCache *lru.Cache[string, *DatabaseMessage]
- projectCache *lru.Cache[string, *ProjectMessage]
- policyCache *lru.Cache[string, *PolicyMessage]
- settingCache *lru.Cache[storepb.SettingName, *SettingMessage]
- rolesCache *expirable.LRU[string, *RoleMessage]
- groupCache *lru.Cache[string, *GroupMessage]
+ Secret string
+ userEmailCache *lru.Cache[string, *UserMessage]
+ instanceCache *lru.Cache[string, *InstanceMessage]
+ databaseCache *lru.Cache[string, *DatabaseMessage]
+ projectCache *lru.Cache[string, *ProjectMessage]
+ policyCache *lru.Cache[string, *PolicyMessage]
+ settingCache *lru.Cache[storepb.SettingName, *SettingMessage]
+ rolesCache *expirable.LRU[string, *RoleMessage]
+ groupCache *expirable.LRU[string, *GroupMessage]
+ groupMembersCache *expirable.LRU[string, map[string]bool]
+ memberGroupsCache *expirable.LRU[string, []string]
// Large objects.
sheetFullCache *lru.Cache[string, *SheetMessage]
@@ -65,10 +67,9 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {
if err != nil {
return nil, err
}
- groupCache, err := lru.New[string, *GroupMessage](1024)
- if err != nil {
- return nil, err
- }
+ groupCache := expirable.NewLRU[string, *GroupMessage](1024, nil, time.Minute)
+ groupMembersCache := expirable.NewLRU[string, map[string]bool](1024, nil, time.Minute)
+ memberGroupsCache := expirable.NewLRU[string, []string](4096, nil, time.Minute)
// Initialize database connection (handles both direct URL and file-based)
dbConnManager := NewDBConnectionManager(pgURL)
@@ -81,15 +82,17 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {
enableCache: enableCache,
// Cache.
- userEmailCache: userEmailCache,
- instanceCache: instanceCache,
- databaseCache: databaseCache,
- projectCache: projectCache,
- policyCache: policyCache,
- settingCache: settingCache,
- rolesCache: rolesCache,
- sheetFullCache: sheetFullCache,
- groupCache: groupCache,
+ userEmailCache: userEmailCache,
+ instanceCache: instanceCache,
+ databaseCache: databaseCache,
+ projectCache: projectCache,
+ policyCache: policyCache,
+ settingCache: settingCache,
+ rolesCache: rolesCache,
+ sheetFullCache: sheetFullCache,
+ groupCache: groupCache,
+ groupMembersCache: groupMembersCache,
+ memberGroupsCache: memberGroupsCache,
}
return s, nil
@@ -111,6 +114,13 @@ func (s *Store) DeleteCache() {
s.userEmailCache.Purge()
}
+// PurgeGroupCaches purges all group-related caches.
+func (s *Store) PurgeGroupCaches() {
+ s.groupCache.Purge()
+ s.groupMembersCache.Purge()
+ s.memberGroupsCache.Purge()
+}
+
func getInstanceCacheKey(instanceID string) string {
return instanceID
}