Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(store): consolidate listX and getX functions (#18354)
Danny Xu7665cb655c68
diff --git a/backend/api/v1/project_service.go b/backend/api/v1/project_service.goindex 63a7d7721e..00a8956b6a 100644--- a/backend/api/v1/project_service.go+++ b/backend/api/v1/project_service.go@@ -1092,7 +1092,7 @@ func validateIAMPolicy(maximumRoleExpiration = workspaceProfileSetting.MaximumRoleExpiration}- roleMessages, err := stores.ListRoles(ctx)+ roleMessages, err := stores.ListRoles(ctx, &store.FindRoleMessage{})if err != nil {return false, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to list roles"))}diff --git a/backend/api/v1/role_service.go b/backend/api/v1/role_service.goindex 5321f0ac9d..28cb1f96df 100644--- a/backend/api/v1/role_service.go+++ b/backend/api/v1/role_service.go@@ -37,7 +37,7 @@ func NewRoleService(store *store.Store, iamManager *iam.Manager, licenseService// ListRoles lists roles.func (s *RoleService) ListRoles(ctx context.Context, _ *connect.Request[v1pb.ListRolesRequest]) (*connect.Response[v1pb.ListRolesResponse], error) {- roleMessages, err := s.store.ListRoles(ctx)+ roleMessages, err := s.store.ListRoles(ctx, &store.FindRoleMessage{})if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to list roles"))}@@ -59,7 +59,7 @@ func (s *RoleService) GetRole(ctx context.Context, req *connect.Request[v1pb.Getif err != nil {return nil, connect.NewError(connect.CodeInvalidArgument, err)}- role, err := s.store.GetRole(ctx, roleID)+ role, err := s.store.GetRole(ctx, &store.FindRoleMessage{ResourceID: &roleID})if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to get role"))}@@ -139,7 +139,7 @@ func (s *RoleService) UpdateRole(ctx context.Context, req *connect.Request[v1pb.if predefinedRole := s.getBuildinRole(roleID); predefinedRole != nil {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("cannot change the build-in role %s", req.Msg.Role.Name))}- role, err := s.store.GetRole(ctx, roleID)+ role, err := s.store.GetRole(ctx, &store.FindRoleMessage{ResourceID: &roleID})if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to get role"))}@@ -200,7 +200,7 @@ func (s *RoleService) DeleteRole(ctx context.Context, req *connect.Request[v1pb.if predefinedRole := s.getBuildinRole(roleID); predefinedRole != nil {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("cannot delete the build-in role %s", req.Msg.Name))}- role, err := s.store.GetRole(ctx, roleID)+ role, err := s.store.GetRole(ctx, &store.FindRoleMessage{ResourceID: &roleID})if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to get role"))}diff --git a/backend/component/iam/manager.go b/backend/component/iam/manager.goindex 6f9245be80..0d59cdeec9 100644--- a/backend/component/iam/manager.go+++ b/backend/component/iam/manager.go@@ -88,7 +88,7 @@ func (m *Manager) CheckPermission(ctx context.Context, p Permission, user *store}func (m *Manager) ReloadCache(ctx context.Context) error {- roles, err := m.store.ListRoles(ctx)+ roles, err := m.store.ListRoles(ctx, &store.FindRoleMessage{})if err != nil {return err}diff --git a/backend/component/webhook/manager.go b/backend/component/webhook/manager.goindex 21649db393..21a727d93a 100644--- a/backend/component/webhook/manager.go+++ b/backend/component/webhook/manager.go@@ -45,7 +45,7 @@ func NewManager(store *store.Store, iamManager *iam.Manager, profile *config.Pro}func (m *Manager) CreateEvent(ctx context.Context, e *Event) {- webhookList, err := m.store.FindProjectWebhook(ctx, &store.FindProjectWebhookMessage{+ webhookList, err := m.store.ListProjectWebhooks(ctx, &store.FindProjectWebhookMessage{ProjectID: &e.Project.ResourceID,EventType: &e.Type,})diff --git a/backend/store/database.go b/backend/store/database.goindex e097cb460d..ffeead3ce4 100644--- a/backend/store/database.go+++ b/backend/store/database.go@@ -89,7 +89,7 @@ func (s *Store) GetDatabase(ctx context.Context, find *FindDatabaseMessage) (*Da}defer tx.Rollback()- databases, err := s.listDatabaseImpl(ctx, tx, find)+ databases, err := s.ListDatabases(ctx, find)if err != nil {return nil, err}@@ -117,10 +117,115 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([}defer tx.Rollback()- databases, err := s.listDatabaseImpl(ctx, tx, find)+ from := qb.Q().Space("db")+ where := qb.Q().Space("TRUE")++ if filterQ := find.FilterQ; filterQ != nil {+ where.And("?", filterQ)+ // Check if the filter requires the db_schema table for table filtering+ sql, _, err := filterQ.ToSQL()+ if err == nil && strings.Contains(sql, "ds.metadata") {+ from.Space("INNER JOIN db_schema ds ON db.instance = ds.instance AND db.name = ds.db_name")+ }+ }++ from.Space("LEFT JOIN instance ON db.instance = instance.resource_id")++ if v := find.ProjectID; v != nil {+ where.And("db.project = ?", *v)+ }+ if v := find.EffectiveEnvironmentID; v != nil {+ where.And(`COALESCE(+ db.environment,+ instance.environment+ ) = ?`, *v)+ }+ if v := find.InstanceID; v != nil {+ where.And("db.instance = ?", *v)+ }+ if v := find.DatabaseName; v != nil {+ if find.IsCaseSensitive {+ where.And("db.name = ?", *v)+ } else {+ where.And("LOWER(db.name) = LOWER(?)", *v)+ }+ }+ if v := find.Engine; v != nil {+ where.And("instance.metadata->>'engine' = ?", *v)+ }+ if !find.ShowDeleted {+ where.And("instance.deleted = ?", false)+ where.And("db.deleted = ?", false)+ }++ q := qb.Q().Space(`+ SELECT+ db.project,+ COALESCE(+ db.environment,+ instance.environment+ ),+ db.environment,+ db.instance,+ db.name,+ db.deleted,+ db.metadata+ FROM ?+ WHERE ?+ ORDER BY db.project, db.instance, db.name+ `, from, where)++ if v := find.Limit; v != nil {+ q.Space("LIMIT ?", *v)+ }+ if v := find.Offset; v != nil {+ q.Space("OFFSET ?", *v)+ }++ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql %+v", q)+ }++ var databases []*DatabaseMessage+ rows, err := tx.QueryContext(ctx, query, args...)if err != nil {return nil, err}+ defer rows.Close()+ for rows.Next() {+ databaseMessage := &DatabaseMessage{}+ var metadataString string+ var effectiveEnvironment, environment sql.NullString+ if err := rows.Scan(+ &databaseMessage.ProjectID,+ &effectiveEnvironment,+ &environment,+ &databaseMessage.InstanceID,+ &databaseMessage.DatabaseName,+ &databaseMessage.Deleted,+ &metadataString,+ ); err != nil {+ return nil, err+ }+ if effectiveEnvironment.Valid {+ databaseMessage.EffectiveEnvironmentID = &effectiveEnvironment.String+ }+ if environment.Valid {+ databaseMessage.EnvironmentID = &environment.String+ }++ var metadata storepb.DatabaseMetadata+ if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(metadataString), &metadata); err != nil {+ return nil, err+ }+ databaseMessage.Metadata = &metadata++ databases = append(databases, databaseMessage)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }if err := tx.Commit(); err != nil {return nil, err@@ -395,120 +500,6 @@ func (s *Store) BatchUpdateDatabases(ctx context.Context, databases []*DatabaseMreturn updatedDatabases, nil}-func (*Store) listDatabaseImpl(ctx context.Context, txn *sql.Tx, find *FindDatabaseMessage) ([]*DatabaseMessage, error) {- from := qb.Q().Space("db")- where := qb.Q().Space("TRUE")-- if filterQ := find.FilterQ; filterQ != nil {- where.And("?", filterQ)- // Check if the filter requires the db_schema table for table filtering- sql, _, err := filterQ.ToSQL()- if err == nil && strings.Contains(sql, "ds.metadata") {- from.Space("INNER JOIN db_schema ds ON db.instance = ds.instance AND db.name = ds.db_name")- }- }-- from.Space("LEFT JOIN instance ON db.instance = instance.resource_id")-- if v := find.ProjectID; v != nil {- where.And("db.project = ?", *v)- }- if v := find.EffectiveEnvironmentID; v != nil {- where.And(`COALESCE(- db.environment,- instance.environment- ) = ?`, *v)- }- if v := find.InstanceID; v != nil {- where.And("db.instance = ?", *v)- }- if v := find.DatabaseName; v != nil {- if find.IsCaseSensitive {- where.And("db.name = ?", *v)- } else {- where.And("LOWER(db.name) = LOWER(?)", *v)- }- }- if v := find.Engine; v != nil {- where.And("instance.metadata->>'engine' = ?", *v)- }- if !find.ShowDeleted {- where.And("instance.deleted = ?", false)- where.And("db.deleted = ?", false)- }-- q := qb.Q().Space(`- SELECT- db.project,- COALESCE(- db.environment,- instance.environment- ),- db.environment,- db.instance,- db.name,- db.deleted,- db.metadata- FROM ?- WHERE ?- ORDER BY db.project, db.instance, db.name- `, from, where)-- if v := find.Limit; v != nil {- q.Space("LIMIT ?", *v)- }- if v := find.Offset; v != nil {- q.Space("OFFSET ?", *v)- }-- query, args, err := q.ToSQL()- if err != nil {- return nil, errors.Wrapf(err, "failed to build sql %+v", q)- }-- var databaseMessages []*DatabaseMessage- rows, err := txn.QueryContext(ctx, query, args...)- if err != nil {- return nil, err- }- defer rows.Close()- for rows.Next() {- databaseMessage := &DatabaseMessage{}- var metadataString string- var effectiveEnvironment, environment sql.NullString- if err := rows.Scan(- &databaseMessage.ProjectID,- &effectiveEnvironment,- &environment,- &databaseMessage.InstanceID,- &databaseMessage.DatabaseName,- &databaseMessage.Deleted,- &metadataString,- ); err != nil {- return nil, err- }- if effectiveEnvironment.Valid {- databaseMessage.EffectiveEnvironmentID = &effectiveEnvironment.String- }- if environment.Valid {- databaseMessage.EnvironmentID = &environment.String- }-- var metadata storepb.DatabaseMetadata- if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(metadataString), &metadata); err != nil {- return nil, err- }- databaseMessage.Metadata = &metadata-- databaseMessages = append(databaseMessages, databaseMessage)- }- if err := rows.Err(); err != nil {- return nil, err- }-- return databaseMessages, nil-}-func GetListDatabaseFilter(filter string) (*qb.Query, error) {if filter == "" {return nil, nildiff --git a/backend/store/database_group.go b/backend/store/database_group.goindex 5c4b4515fe..2af72d5cc7 100644--- a/backend/store/database_group.go+++ b/backend/store/database_group.go@@ -63,54 +63,6 @@ func (s *Store) ListDatabaseGroups(ctx context.Context, find *FindDatabaseGroupM}defer tx.Rollback()- databaseGroups, err := s.listDatabaseGroupImpl(ctx, tx, find)- if err != nil {- return nil, errors.Wrapf(err, "failed to list database groups")- }-- if err := tx.Commit(); err != nil {- return nil, errors.Wrapf(err, "failed to commit transaction")- }- for _, databaseGroup := range databaseGroups {- s.databaseGroupCache.Add(getDatabaseGroupCacheKey(databaseGroup.ProjectID, databaseGroup.ResourceID), databaseGroup)- }-- return databaseGroups, nil-}--// GetDatabaseGroup gets a database group.-func (s *Store) GetDatabaseGroup(ctx context.Context, find *FindDatabaseGroupMessage) (*DatabaseGroupMessage, error) {- if find.ProjectID != nil && find.ResourceID != nil {- if v, ok := s.databaseGroupCache.Get(getDatabaseGroupCacheKey(*find.ProjectID, *find.ResourceID)); ok && s.enableCache {- return v, nil- }- }- tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})- if err != nil {- return nil, err- }- defer tx.Rollback()-- databaseGroups, err := s.listDatabaseGroupImpl(ctx, tx, find)- if err != nil {- return nil, errors.Wrapf(err, "failed to list database groups")- }-- if err := tx.Commit(); err != nil {- return nil, errors.Wrapf(err, "failed to commit transaction")- }-- if len(databaseGroups) == 0 {- return nil, nil- }- if len(databaseGroups) > 1 {- return nil, errors.Errorf("found multiple database groups")- }- s.databaseGroupCache.Add(getDatabaseGroupCacheKey(databaseGroups[0].ProjectID, databaseGroups[0].ResourceID), databaseGroups[0])- return databaseGroups[0], nil-}--func (*Store) listDatabaseGroupImpl(ctx context.Context, txn *sql.Tx, find *FindDatabaseGroupMessage) ([]*DatabaseGroupMessage, error) {q := qb.Q().Space(`SELECTproject,@@ -136,7 +88,7 @@ func (*Store) listDatabaseGroupImpl(ctx context.Context, txn *sql.Tx, find *Find}var databaseGroups []*DatabaseGroupMessage- rows, err := txn.QueryContext(ctx, query, args...)+ rows, err := tx.QueryContext(ctx, query, args...)if err != nil {return nil, errors.Wrapf(err, "failed to scan")}@@ -163,9 +115,49 @@ func (*Store) listDatabaseGroupImpl(ctx context.Context, txn *sql.Tx, find *Findif err := rows.Err(); err != nil {return nil, errors.Wrapf(err, "failed to scan")}++ if err := tx.Commit(); err != nil {+ return nil, errors.Wrapf(err, "failed to commit transaction")+ }+ for _, databaseGroup := range databaseGroups {+ s.databaseGroupCache.Add(getDatabaseGroupCacheKey(databaseGroup.ProjectID, databaseGroup.ResourceID), databaseGroup)+ }+return databaseGroups, nil}+// GetDatabaseGroup gets a database group.+func (s *Store) GetDatabaseGroup(ctx context.Context, find *FindDatabaseGroupMessage) (*DatabaseGroupMessage, error) {+ if find.ProjectID != nil && find.ResourceID != nil {+ if v, ok := s.databaseGroupCache.Get(getDatabaseGroupCacheKey(*find.ProjectID, *find.ResourceID)); ok && s.enableCache {+ return v, nil+ }+ }+ tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})+ if err != nil {+ return nil, err+ }+ defer tx.Rollback()++ databaseGroups, err := s.ListDatabaseGroups(ctx, find)+ if err != nil {+ return nil, err+ }++ if err := tx.Commit(); err != nil {+ return nil, errors.Wrapf(err, "failed to commit transaction")+ }++ if len(databaseGroups) == 0 {+ return nil, nil+ }+ if len(databaseGroups) > 1 {+ return nil, errors.Errorf("found multiple database groups")+ }+ s.databaseGroupCache.Add(getDatabaseGroupCacheKey(databaseGroups[0].ProjectID, databaseGroups[0].ResourceID), databaseGroups[0])+ return databaseGroups[0], nil+}+// UpdateDatabaseGroup updates a database group.func (s *Store) UpdateDatabaseGroup(ctx context.Context, projectID, resourceID string, patch *UpdateDatabaseGroupMessage) (*DatabaseGroupMessage, error) {set := qb.Q()diff --git a/backend/store/group.go b/backend/store/group.goindex 8dc17a1100..9f172c8f31 100644--- a/backend/store/group.go+++ b/backend/store/group.go@@ -63,7 +63,7 @@ func (s *Store) GetGroup(ctx context.Context, find *FindGroupMessage) (*GroupMes}defer tx.Rollback()- groups, err := s.listGroupImpl(ctx, tx, find)+ groups, err := s.ListGroups(ctx, find)if err != nil {return nil, err}@@ -87,21 +87,6 @@ func (s *Store) ListGroups(ctx context.Context, find *FindGroupMessage) ([]*Grou}defer tx.Rollback()- groups, err := s.listGroupImpl(ctx, tx, find)- if err != nil {- return nil, err- }- if err := tx.Commit(); err != nil {- return nil, err- }-- for _, group := range groups {- s.groupCache.Add(group.Email, group)- }- return groups, nil-}--func (*Store) listGroupImpl(ctx context.Context, txn *sql.Tx, find *FindGroupMessage) ([]*GroupMessage, error) {with := qb.Q()from := qb.Q().Space("user_group")where := qb.Q().Space("TRUE")@@ -160,7 +145,7 @@ func (*Store) listGroupImpl(ctx context.Context, txn *sql.Tx, find *FindGroupMes}var groups []*GroupMessage- rows, err := txn.QueryContext(ctx, query, args...)+ rows, err := tx.QueryContext(ctx, query, args...)if err != nil {return nil, err}@@ -192,7 +177,13 @@ func (*Store) listGroupImpl(ctx context.Context, txn *sql.Tx, find *FindGroupMesif err := rows.Err(); err != nil {return nil, err}+ if err := tx.Commit(); err != nil {+ return nil, err+ }+ for _, group := range groups {+ s.groupCache.Add(group.Email, group)+ }return groups, nil}diff --git a/backend/store/idp.go b/backend/store/idp.goindex ab7e01d3c3..87ffb63002 100644--- a/backend/store/idp.go+++ b/backend/store/idp.go@@ -111,7 +111,7 @@ func (s *Store) GetIdentityProvider(ctx context.Context, find *FindIdentityProvi}defer tx.Rollback()- identityProviders, err := s.listIdentityProvidersImpl(ctx, tx, find)+ identityProviders, err := s.ListIdentityProviders(ctx, find)if err != nil {return nil, err}@@ -138,18 +138,63 @@ func (s *Store) ListIdentityProviders(ctx context.Context, find *FindIdentityPro}defer tx.Rollback()- identityProviders, err := s.listIdentityProvidersImpl(ctx, tx, find)+ q := qb.Q().Space(`+ SELECT+ resource_id,+ name,+ domain,+ type,+ config+ FROM idp+ WHERE TRUE+ `)++ if v := find.ResourceID; v != nil {+ q.And("resource_id = ?", *v)+ }++ q.Space("ORDER BY id ASC")++ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }++ rows, err := tx.QueryContext(ctx, query, args...)if err != nil {return nil, err}+ defer rows.Close()++ var identityProviderMessages []*IdentityProviderMessage+ for rows.Next() {+ var identityProviderMessage IdentityProviderMessage+ var identityProviderType string+ var identityProviderConfig string+ if err := rows.Scan(+ &identityProviderMessage.ResourceID,+ &identityProviderMessage.Title,+ &identityProviderMessage.Domain,+ &identityProviderType,+ &identityProviderConfig,+ ); err != nil {+ return nil, err+ }+ identityProviderMessage.Type = convertIdentityProviderType(identityProviderType)+ identityProviderMessage.Config = convertIdentityProviderConfigString(identityProviderMessage.Type, identityProviderConfig)+ identityProviderMessages = append(identityProviderMessages, &identityProviderMessage)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }if err := tx.Commit(); err != nil {return nil, err}- for _, identityProvider := range identityProviders {+ for _, identityProvider := range identityProviderMessages {s.idpCache.Add(identityProvider.ResourceID, identityProvider)}- return identityProviders, nil+ return identityProviderMessages, nil}// UpdateIdentityProvider updates an identity provider.@@ -229,60 +274,6 @@ func (*Store) updateIdentityProviderImpl(ctx context.Context, txn *sql.Tx, patchreturn identityProvider, nil}-func (*Store) listIdentityProvidersImpl(ctx context.Context, txn *sql.Tx, find *FindIdentityProviderMessage) ([]*IdentityProviderMessage, error) {- q := qb.Q().Space(`- SELECT- resource_id,- name,- domain,- type,- config- FROM idp- WHERE TRUE- `)-- if v := find.ResourceID; v != nil {- q.And("resource_id = ?", *v)- }-- q.Space("ORDER BY id ASC")-- query, args, err := q.ToSQL()- if err != nil {- return nil, errors.Wrapf(err, "failed to build sql")- }-- rows, err := txn.QueryContext(ctx, query, args...)- if err != nil {- return nil, err- }- defer rows.Close()-- var identityProviderMessages []*IdentityProviderMessage- for rows.Next() {- var identityProviderMessage IdentityProviderMessage- var identityProviderType string- var identityProviderConfig string- if err := rows.Scan(- &identityProviderMessage.ResourceID,- &identityProviderMessage.Title,- &identityProviderMessage.Domain,- &identityProviderType,- &identityProviderConfig,- ); err != nil {- return nil, err- }- identityProviderMessage.Type = convertIdentityProviderType(identityProviderType)- identityProviderMessage.Config = convertIdentityProviderConfigString(identityProviderMessage.Type, identityProviderConfig)- identityProviderMessages = append(identityProviderMessages, &identityProviderMessage)- }- if err := rows.Err(); err != nil {- return nil, err- }-- return identityProviderMessages, nil-}-func (s *Store) DeleteIdentityProvider(ctx context.Context, resourceID string) error {tx, err := s.GetDB().BeginTx(ctx, nil)if err != nil {diff --git a/backend/store/policy.go b/backend/store/policy.goindex 29c2909ea0..6805d11bb4 100644--- a/backend/store/policy.go+++ b/backend/store/policy.go@@ -432,7 +432,7 @@ func (s *Store) GetPolicy(ctx context.Context, find *FindPolicyMessage) (*Policy// We will always return the resource regardless of its deleted state.find.ShowAll = true- policies, err := s.listPolicyImpl(ctx, tx, find)+ policies, err := s.ListPolicies(ctx, find)if err != nil {return nil, err}@@ -465,20 +465,83 @@ func (s *Store) ListPolicies(ctx context.Context, find *FindPolicyMessage) ([]*P}defer tx.Rollback()- policies, err := s.listPolicyImpl(ctx, tx, find)+ q := qb.Q().Space(`+ SELECT+ updated_at,+ resource_type,+ resource,+ inherit_from_parent,+ type,+ payload,+ enforce+ FROM policy+ WHERE TRUE+ `)++ if v := find.ResourceType; v != nil {+ q.And("resource_type = ?", v.String())+ }+ if v := find.Resource; v != nil {+ q.And("resource = ?", *v)+ }+ if v := find.Type; v != nil {+ q.And("type = ?", v.String())+ }+ if !find.ShowAll {+ q.And("enforce = ?", true)+ }++ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }++ rows, err := tx.QueryContext(ctx, query, args...)if err != nil {return nil, err}+ defer rows.Close()++ var policyList []*PolicyMessage+ for rows.Next() {+ var policyMessage PolicyMessage+ var resourceTypeString, typeString string+ if err := rows.Scan(+ &policyMessage.UpdatedAt,+ &resourceTypeString,+ &policyMessage.Resource,+ &policyMessage.InheritFromParent,+ &typeString,+ &policyMessage.Payload,+ &policyMessage.Enforce,+ ); err != nil {+ return nil, err+ }+ resourceTypeValue, ok := storepb.Policy_Resource_value[resourceTypeString]+ if !ok {+ return nil, errors.Errorf("invalid policy resource type string: %s", resourceTypeString)+ }+ policyMessage.ResourceType = storepb.Policy_Resource(resourceTypeValue)+ value, ok := storepb.Policy_Type_value[typeString]+ if !ok {+ return nil, errors.Errorf("invalid policy type string: %s", typeString)+ }+ policyMessage.Type = storepb.Policy_Type(value)+ policyList = append(policyList, &policyMessage)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }if err := tx.Commit(); err != nil {return nil, err}- for _, policy := range policies {+ for _, policy := range policyList {s.policyCache.Add(getPolicyCacheKey(policy.ResourceType, policy.Resource, policy.Type), policy)}- return policies, nil+ return policyList, nil}// CreatePolicy creates a policy.@@ -625,74 +688,3 @@ func upsertPolicyImpl(ctx context.Context, txn *sql.Tx, create *PolicyMessage) (}return create, nil}--func (*Store) listPolicyImpl(ctx context.Context, txn *sql.Tx, find *FindPolicyMessage) ([]*PolicyMessage, error) {- q := qb.Q().Space(`- SELECT- updated_at,- resource_type,- resource,- inherit_from_parent,- type,- payload,- enforce- FROM policy- WHERE TRUE- `)-- if v := find.ResourceType; v != nil {- q.And("resource_type = ?", v.String())- }- if v := find.Resource; v != nil {- q.And("resource = ?", *v)- }- if v := find.Type; v != nil {- q.And("type = ?", v.String())- }- if !find.ShowAll {- q.And("enforce = ?", true)- }-- query, args, err := q.ToSQL()- if err != nil {- return nil, errors.Wrapf(err, "failed to build sql")- }-- rows, err := txn.QueryContext(ctx, query, args...)- if err != nil {- return nil, err- }- defer rows.Close()-- var policyList []*PolicyMessage- for rows.Next() {- var policyMessage PolicyMessage- var resourceTypeString, typeString string- if err := rows.Scan(- &policyMessage.UpdatedAt,- &resourceTypeString,- &policyMessage.Resource,- &policyMessage.InheritFromParent,- &typeString,- &policyMessage.Payload,- &policyMessage.Enforce,- ); err != nil {- return nil, err- }- resourceTypeValue, ok := storepb.Policy_Resource_value[resourceTypeString]- if !ok {- return nil, errors.Errorf("invalid policy resource type string: %s", resourceTypeString)- }- policyMessage.ResourceType = storepb.Policy_Resource(resourceTypeValue)- value, ok := storepb.Policy_Type_value[typeString]- if !ok {- return nil, errors.Errorf("invalid policy type string: %s", typeString)- }- policyMessage.Type = storepb.Policy_Type(value)- policyList = append(policyList, &policyMessage)- }- if err := rows.Err(); err != nil {- return nil, err- }- return policyList, nil-}diff --git a/backend/store/project.go b/backend/store/project.goindex 873aad79d7..4da60b81e6 100644--- a/backend/store/project.go+++ b/backend/store/project.go@@ -70,7 +70,7 @@ func (s *Store) GetProject(ctx context.Context, find *FindProjectMessage) (*Proj}defer tx.Rollback()- projects, err := s.listProjectImpl(ctx, tx, find)+ projects, err := s.ListProjects(ctx, find)if err != nil {return nil, err}@@ -98,19 +98,75 @@ func (s *Store) ListProjects(ctx context.Context, find *FindProjectMessage) ([]*}defer tx.Rollback()- projects, err := s.listProjectImpl(ctx, tx, find)+ q := qb.Q().Space("SELECT resource_id, name, data_classification_config_id, setting, deleted FROM project WHERE TRUE")+ if filterQ := find.FilterQ; filterQ != nil {+ q.And("?", filterQ)+ }+ if v := find.ResourceID; v != nil {+ q.And("resource_id = ?", *v)+ }+ if !find.ShowDeleted {+ q.And("deleted = ?", false)+ }+ q.Space("ORDER BY project.resource_id")+ if v := find.Limit; v != nil {+ q.Space("LIMIT ?", *v)+ }+ if v := find.Offset; v != nil {+ q.Space("OFFSET ?", *v)+ }++ sql, args, err := q.ToSQL()if err != nil {return nil, err}+ var projectMessages []*ProjectMessage+ rows, err := tx.QueryContext(ctx, sql, args...)+ if err != nil {+ return nil, err+ }+ defer rows.Close()++ for rows.Next() {+ var projectMessage ProjectMessage+ var payload []byte+ if err := rows.Scan(+ &projectMessage.ResourceID,+ &projectMessage.Title,+ &projectMessage.DataClassificationConfigID,+ &payload,+ &projectMessage.Deleted,+ ); err != nil {+ return nil, err+ }+ setting := &storepb.Project{}+ if err := common.ProtojsonUnmarshaler.Unmarshal(payload, setting); err != nil {+ return nil, err+ }+ projectMessage.Setting = setting+ projectMessages = append(projectMessages, &projectMessage)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }++ for _, project := range projectMessages {+ projectWebhooks, err := s.ListProjectWebhooks(ctx, &FindProjectWebhookMessage{ProjectID: &project.ResourceID})+ if err != nil {+ return nil, err+ }+ project.Webhooks = projectWebhooks+ }+if err := tx.Commit(); err != nil {return nil, err}- for _, project := range projects {+ for _, project := range projectMessages {s.storeProjectCache(project)}- return projects, nil+ return projectMessages, nil}// CreateProject creates a project.@@ -283,71 +339,6 @@ func updateProjectImpl(ctx context.Context, txn *sql.Tx, patch *UpdateProjectMesreturn nil}-func (s *Store) listProjectImpl(ctx context.Context, txn *sql.Tx, find *FindProjectMessage) ([]*ProjectMessage, error) {- q := qb.Q().Space("SELECT resource_id, name, data_classification_config_id, setting, deleted FROM project WHERE TRUE")- if filterQ := find.FilterQ; filterQ != nil {- q.And("?", filterQ)- }- if v := find.ResourceID; v != nil {- q.And("resource_id = ?", *v)- }- if !find.ShowDeleted {- q.And("deleted = ?", false)- }- q.Space("ORDER BY project.resource_id")- if v := find.Limit; v != nil {- q.Space("LIMIT ?", *v)- }- if v := find.Offset; v != nil {- q.Space("OFFSET ?", *v)- }-- sql, args, err := q.ToSQL()- if err != nil {- return nil, err- }-- var projectMessages []*ProjectMessage- rows, err := txn.QueryContext(ctx, sql, args...)- if err != nil {- return nil, err- }- defer rows.Close()-- for rows.Next() {- var projectMessage ProjectMessage- var payload []byte- if err := rows.Scan(- &projectMessage.ResourceID,- &projectMessage.Title,- &projectMessage.DataClassificationConfigID,- &payload,- &projectMessage.Deleted,- ); err != nil {- return nil, err- }- setting := &storepb.Project{}- if err := common.ProtojsonUnmarshaler.Unmarshal(payload, setting); err != nil {- return nil, err- }- projectMessage.Setting = setting- projectMessages = append(projectMessages, &projectMessage)- }- if err := rows.Err(); err != nil {- return nil, err- }-- for _, project := range projectMessages {- projectWebhooks, err := s.findProjectWebhookImpl(ctx, txn, &FindProjectWebhookMessage{ProjectID: &project.ResourceID})- if err != nil {- return nil, err- }- project.Webhooks = projectWebhooks- }-- return projectMessages, nil-}-func (s *Store) storeProjectCache(project *ProjectMessage) {s.projectCache.Add(project.ResourceID, project)}diff --git a/backend/store/project_webhook.go b/backend/store/project_webhook.goindex 74d03c8d45..4e89dc60c5 100644--- a/backend/store/project_webhook.go+++ b/backend/store/project_webhook.go@@ -87,17 +87,81 @@ func (s *Store) CreateProjectWebhook(ctx context.Context, projectID string, creareturn &projectWebhook, nil}-// FindProjectWebhook finds a list of ProjectWebhook instances.-func (s *Store) FindProjectWebhook(ctx context.Context, find *FindProjectWebhookMessage) ([]*ProjectWebhookMessage, error) {+// ListProjectWebhooks lists a list of ProjectWebhook instances.+func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhookMessage) ([]*ProjectWebhookMessage, error) {tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})if err != nil {return nil, errors.Wrapf(err, "failed to begin transaction")}- webhooks, err := s.findProjectWebhookImpl(ctx, tx, find)+ q := qb.Q().Space(`+ SELECT+ id,+ project,+ payload+ FROM project_webhook+ WHERE TRUE+ `)++ if v := find.ID; v != nil {+ q.And("id = ?", *v)+ }+ if v := find.ProjectID; v != nil {+ q.And("project = ?", *v)+ }+ if v := find.URL; v != nil {+ q.And("payload->>'url' = ?", *v)+ }++ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }++ rows, err := tx.QueryContext(ctx, query, args...)if err != nil {return nil, err}+ defer rows.Close()++ var projectWebhooks []*ProjectWebhookMessage // Declare it here+ for rows.Next() {+ projectWebhook := ProjectWebhookMessage{+ Payload: &storepb.ProjectWebhook{},+ }+ var payload []byte++ if err := rows.Scan(+ &projectWebhook.ID,+ &projectWebhook.ProjectID,+ &payload,+ ); err != nil {+ return nil, err+ }++ if err := common.ProtojsonUnmarshaler.Unmarshal(payload, projectWebhook.Payload); err != nil {+ return nil, errors.Wrapf(err, "failed to unmarshal")+ }++ if v := find.EventType; v != nil {+ found := false+ for _, activity := range projectWebhook.Payload.Activities {+ if activity == *v {+ found = true+ break+ }+ }+ if found {+ projectWebhooks = append(projectWebhooks, &projectWebhook)+ }+ } else {+ projectWebhooks = append(projectWebhooks, &projectWebhook)+ }+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ webhooks := projectWebhooksif err := tx.Commit(); err != nil {return nil, errors.Wrapf(err, "failed to commit transaction")@@ -112,7 +176,7 @@ func (s *Store) GetProjectWebhook(ctx context.Context, find *FindProjectWebhookMreturn nil, errors.Wrapf(err, "failed to begin transaction")}- webhooks, err := s.findProjectWebhookImpl(ctx, tx, find)+ webhooks, err := s.ListProjectWebhooks(ctx, find)if err != nil {return nil, err}@@ -212,75 +276,3 @@ func (s *Store) DeleteProjectWebhook(ctx context.Context, projectResourceID stris.removeProjectCache(projectResourceID)return nil}--func (*Store) findProjectWebhookImpl(ctx context.Context, txn *sql.Tx, find *FindProjectWebhookMessage) ([]*ProjectWebhookMessage, error) {- q := qb.Q().Space(`- SELECT- id,- project,- payload- FROM project_webhook- WHERE TRUE- `)-- if v := find.ID; v != nil {- q.And("id = ?", *v)- }- if v := find.ProjectID; v != nil {- q.And("project = ?", *v)- }- if v := find.URL; v != nil {- q.And("payload->>'url' = ?", *v)- }-- query, args, err := q.ToSQL()- if err != nil {- return nil, errors.Wrapf(err, "failed to build sql")- }-- rows, err := txn.QueryContext(ctx, query, args...)- if err != nil {- return nil, err- }- defer rows.Close()-- var projectWebhooks []*ProjectWebhookMessage- for rows.Next() {- projectWebhook := ProjectWebhookMessage{- Payload: &storepb.ProjectWebhook{},- }- var payload []byte-- if err := rows.Scan(- &projectWebhook.ID,- &projectWebhook.ProjectID,- &payload,- ); err != nil {- return nil, err- }-- if err := common.ProtojsonUnmarshaler.Unmarshal(payload, projectWebhook.Payload); err != nil {- return nil, errors.Wrapf(err, "failed to unmarshal")- }-- if v := find.EventType; v != nil {- found := false- for _, activity := range projectWebhook.Payload.Activities {- if activity == *v {- found = true- break- }- }- if found {- projectWebhooks = append(projectWebhooks, &projectWebhook)- }- } else {- projectWebhooks = append(projectWebhooks, &projectWebhook)- }- }- if err := rows.Err(); err != nil {- return nil, err- }-- return projectWebhooks, nil-}diff --git a/backend/store/role.go b/backend/store/role.goindex 0c4837e18f..d27a8c9dec 100644--- a/backend/store/role.go+++ b/backend/store/role.go@@ -2,7 +2,6 @@ package storeimport ("context"- "database/sql""github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson"@@ -29,6 +28,11 @@ type UpdateRoleMessage struct {Permissions *map[string]bool}+// FindRoleMessage is the message for finding roles.+type FindRoleMessage struct {+ ResourceID *string+}+type RoleUsedByResource struct {ResourceType storepb.Policy_ResourceResource string@@ -103,53 +107,42 @@ func (s *Store) CreateRole(ctx context.Context, create *RoleMessage) (*RoleMessa}// GetRole returns a role by ID.-func (s *Store) GetRole(ctx context.Context, resourceID string) (*RoleMessage, error) {- if v, ok := s.rolesCache.Get(resourceID); ok && s.enableCache {- return v, nil+func (s *Store) GetRole(ctx context.Context, find *FindRoleMessage) (*RoleMessage, error) {+ if find.ResourceID != nil {+ if v, ok := s.rolesCache.Get(*find.ResourceID); ok && s.enableCache {+ return v, nil+ }}- q := qb.Q().Space(`- SELECT- name, description, permissions- FROM role- WHERE resource_id = ?- `, resourceID)-- query, args, err := q.ToSQL()+ roles, err := s.ListRoles(ctx, find)if err != nil {- return nil, errors.Wrapf(err, "failed to build sql")- }-- role := &RoleMessage{- Permissions: map[string]bool{},- }- var permissions []byte- if err := s.GetDB().QueryRowContext(ctx, query, args...).Scan(&role.Name, &role.Description, &permissions); err != nil {- if err == sql.ErrNoRows {- return nil, nil- }return nil, err}- var rolePermissions storepb.RolePermissions- if err := common.ProtojsonUnmarshaler.Unmarshal(permissions, &rolePermissions); err != nil {- return nil, err+ if len(roles) == 0 {+ return nil, nil}- for _, v := range rolePermissions.Permissions {- role.Permissions[v] = true+ if len(roles) > 1 {+ return nil, &common.Error{Code: common.Conflict, Err: errors.Errorf("found %d roles with filter %+v, expect 1", len(roles), find)}}- role.ResourceID = resourceID- s.rolesCache.Add(resourceID, role)+ role := roles[0]++ s.rolesCache.Add(role.ResourceID, role)return role, nil}// ListRoles returns a list of roles.-func (s *Store) ListRoles(ctx context.Context) ([]*RoleMessage, error) {+func (s *Store) ListRoles(ctx context.Context, find *FindRoleMessage) ([]*RoleMessage, error) {q := qb.Q().Space(`SELECTresource_id, name, description, permissionsFROM role+ WHERE TRUE`)+ if v := find.ResourceID; v != nil {+ q.And("resource_id = ?", *v)+ }+query, args, err := q.ToSQL()if err != nil {return nil, errors.Wrapf(err, "failed to build sql")