Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(qb): user filter (#17852)
p0nyf032dd74c384
diff --git a/backend/api/v1/user_service.go b/backend/api/v1/user_service.goindex fe78c19d7e..b56612a82c 100644--- a/backend/api/v1/user_service.go+++ b/backend/api/v1/user_service.go@@ -8,10 +8,6 @@ import ("strings""connectrpc.com/connect"- "github.com/google/cel-go/cel"- celast "github.com/google/cel-go/common/ast"- celoperators "github.com/google/cel-go/common/operators"- celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors""github.com/pquerna/otp/totp"@@ -124,8 +120,13 @@ func (s *UserService) ListUsers(ctx context.Context, request *connect.Request[v1Offset: &offset.offset,ShowDeleted: request.Msg.ShowDeleted,}- if err := parseListUserFilter(find, request.Msg.Filter); err != nil {- return nil, err+ filterResult, err := store.GetListUserFilter(request.Msg.Filter)+ if err != nil {+ return nil, connect.NewError(connect.CodeInvalidArgument, err)+ }+ if filterResult != nil {+ find.FilterQ = filterResult.Query+ find.ProjectID = filterResult.ProjectID}if v := find.ProjectID; v != nil {user, ok := GetUserFromContext(ctx)@@ -163,145 +164,6 @@ func (s *UserService) ListUsers(ctx context.Context, request *connect.Request[v1return connect.NewResponse(response), nil}-func parseListUserFilter(find *store.FindUserMessage, filter string) error {- if filter == "" {- return nil- }- e, err := cel.NewEnv()- if err != nil {- return connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))- }- ast, iss := e.Parse(filter)- if iss != nil {- return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))- }-- var getFilter func(expr celast.Expr) (string, error)- var positionalArgs []any-- parseToSQL := func(variable, value any) (string, error) {- switch variable {- case "email":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("principal.email = $%d", len(positionalArgs)), nil- case "name":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("principal.name = $%d", len(positionalArgs)), nil- case "user_type":- v1UserType, ok := v1pb.UserType_value[value.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid user type filter %q", value))- }- principalType, err := convertToPrincipalType(v1pb.UserType(v1UserType))- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse the user type %q with error: %v", v1UserType, err))- }- positionalArgs = append(positionalArgs, principalType)- return fmt.Sprintf("principal.type = $%d", len(positionalArgs)), nil- case "state":- v1State, ok := v1pb.State_value[value.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid state filter %q", value))- }- positionalArgs = append(positionalArgs, v1pb.State(v1State) == v1pb.State_DELETED)- return fmt.Sprintf("principal.deleted = $%d", len(positionalArgs)), nil- case "project":- projectID, err := common.GetProjectID(value.(string))- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid project filter %q", value))- }- find.ProjectID = &projectID- return "TRUE", nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- }-- parseToUserTypeSQL := func(expr celast.Expr, relation string) (string, error) {- variable, value := getVariableAndValueFromExpr(expr)- if variable != "user_type" {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "user_type" support "user_type in [xx]"/"!(user_type in [xx])" operator`))- }-- rawTypeList, ok := value.([]any)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid user_type value %q", value))- }- if len(rawTypeList) == 0 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("empty user_type filter"))- }- userTypeList := []string{}- for _, rawType := range rawTypeList {- v1UserType, ok := v1pb.UserType_value[rawType.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid user type filter %q", rawType))- }- principalType, err := convertToPrincipalType(v1pb.UserType(v1UserType))- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse the user type %q with error: %v", v1UserType, err))- }- positionalArgs = append(positionalArgs, principalType)- userTypeList = append(userTypeList, fmt.Sprintf("$%d", len(positionalArgs)))- }-- return fmt.Sprintf("principal.type %s (%s)", relation, strings.Join(userTypeList, ",")), nil- }-- getFilter = func(expr celast.Expr) (string, error) {- switch expr.Kind() {- case celast.CallKind:- functionName := expr.AsCall().FunctionName()- switch functionName {- case celoperators.LogicalOr:- return getSubConditionFromExpr(expr, getFilter, "OR")- case celoperators.LogicalAnd:- return getSubConditionFromExpr(expr, getFilter, "AND")- case celoperators.Equals:- variable, value := getVariableAndValueFromExpr(expr)- return parseToSQL(variable, value)- case celoverloads.Matches:- variable := expr.AsCall().Target().AsIdent()- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))- }- value := args[0].AsLiteral().Value()- if variable != "name" && variable != "email" {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "name" and "email" support %q operator, but found %q`, celoverloads.Matches, variable))- }- strValue, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))- }- return "LOWER(principal." + variable + ") LIKE '%" + strings.ToLower(strValue) + "%'", nil- case celoperators.In:- return parseToUserTypeSQL(expr, "IN")- case celoperators.LogicalNot:- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only support !(user_type in ["{type1}", "{type2}"]) format`))- }- return parseToUserTypeSQL(args[0], "NOT IN")- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))- }- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected expr kind %v", expr.Kind()))- }- }-- where, err := getFilter(ast.NativeRep().Expr())- if err != nil {- return err- }-- find.Filter = &store.ListResourceFilter{- Args: positionalArgs,- Where: "(" + where + ")",- }- return nil-}-// CreateUser creates a user.func (s *UserService) CreateUser(ctx context.Context, request *connect.Request[v1pb.CreateUserRequest]) (*connect.Response[v1pb.User], error) {if err := s.licenseService.IsFeatureEnabled(v1pb.PlanFeature_FEATURE_DISALLOW_SELF_SERVICE_SIGNUP); err == nil {diff --git a/backend/api/v1/user_service_test.go b/backend/api/v1/user_service_test.goindex c99885e93a..b7b1f993d5 100644--- a/backend/api/v1/user_service_test.go+++ b/backend/api/v1/user_service_test.go@@ -1,78 +1 @@package v1--import (- "strings"- "testing"-- "connectrpc.com/connect"- "github.com/pkg/errors"- "github.com/stretchr/testify/require"-- v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"- "github.com/bytebase/bytebase/backend/store"-)--func TestListUserFilter(t *testing.T) {- testCases := []struct {- input string- want *store.ListResourceFilter- error *connect.Error- }{- {- input: `title == "ed"`,- error: connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", "title")),- },- {- input: `name == "ed"`,- want: &store.ListResourceFilter{- Where: `(principal.name = $1)`,- Args: []any{"ed"},- },- },- {- input: `name.matches("ED")`,- want: &store.ListResourceFilter{- Where: `(LOWER(principal.name) LIKE '%ed%')`,- },- },- {- input: `user_type in ["SERVICE_ACCOUNT", "USER"]`,- want: &store.ListResourceFilter{- Where: `(principal.type IN ($1,$2))`,- Args: []any{v1pb.UserType_SERVICE_ACCOUNT, v1pb.UserType_USER},- },- },- {- input: `state == "DELETED"`,- want: &store.ListResourceFilter{- Where: `(principal.deleted = $1)`,- Args: []any{true},- },- },- {- input: `project == "projects/sample-project"`,- want: &store.ListResourceFilter{- Where: `(TRUE)`,- },- },- }-- for _, tc := range testCases {- find := &store.FindUserMessage{}- err := parseListUserFilter(find, tc.input)- if tc.error != nil {- require.Error(t, err)- connectErr := new(connect.Error)- require.True(t, errors.As(err, &connectErr))- require.Equal(t, tc.error.Message(), connectErr.Message())- require.Equal(t, tc.error.Code(), connectErr.Code())- } else {- require.NoError(t, err)- require.Equal(t, tc.want.Where, find.Filter.Where)-- if strings.HasPrefix(tc.input, "project ==") {- require.NotNil(t, find.ProjectID)- }- }- }-}diff --git a/backend/store/principal.go b/backend/store/principal.goindex 5cfc8ff23a..5885ae40fa 100644--- a/backend/store/principal.go+++ b/backend/store/principal.go@@ -33,7 +33,7 @@ type FindUserMessage struct {Type *storepb.PrincipalTypeLimit *intOffset *int- Filter *ListResourceFilter+ FilterQ *qb.QueryProjectID *string}@@ -244,8 +244,8 @@ func listUserImpl(ctx context.Context, txn *sql.Tx, find *FindUserMessage) ([]*UGROUP BY principal.id)`)- if filter := find.Filter; filter != nil {- where.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)+ if filterQ := find.FilterQ; filterQ != nil {+ where.And("?", filterQ)}if v := find.ID; v != nil {where.And("principal.id = ?", *v)diff --git a/backend/store/principal_filter.go b/backend/store/principal_filter.gonew file mode 100644index 0000000000..e34262d307--- /dev/null+++ b/backend/store/principal_filter.go@@ -0,0 +1,201 @@+package store++import (+ "fmt"+ "strings"++ "github.com/google/cel-go/cel"+ celast "github.com/google/cel-go/common/ast"+ celoperators "github.com/google/cel-go/common/operators"+ celoverloads "github.com/google/cel-go/common/overloads"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/qb"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// GetListUserFilterResult contains the filter query and optional project ID.+type GetListUserFilterResult struct {+ Query *qb.Query+ ProjectID *string+}++// GetListUserFilter parses a CEL filter string and returns a query for filtering users.+func GetListUserFilter(filter string) (*GetListUserFilterResult, error) {+ if filter == "" {+ return nil, nil+ }++ e, err := cel.NewEnv()+ if err != nil {+ return nil, errors.Errorf("failed to create cel env")+ }+ ast, iss := e.Parse(filter)+ if iss != nil {+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())+ }++ var getFilter func(expr celast.Expr) (*qb.Query, error)+ var projectID *string++ convertToPrincipalType := func(v1UserType v1pb.UserType) (storepb.PrincipalType, error) {+ switch v1UserType {+ case v1pb.UserType_USER:+ return storepb.PrincipalType_END_USER, nil+ case v1pb.UserType_SYSTEM_BOT:+ return storepb.PrincipalType_SYSTEM_BOT, nil+ case v1pb.UserType_SERVICE_ACCOUNT:+ return storepb.PrincipalType_SERVICE_ACCOUNT, nil+ default:+ return storepb.PrincipalType_END_USER, errors.Errorf("invalid user type %s", v1UserType)+ }+ }++ parseToSQL := func(variable, value any) (*qb.Query, error) {+ switch variable {+ case "email":+ return qb.Q().Space("principal.email = ?", value.(string)), nil+ case "name":+ return qb.Q().Space("principal.name = ?", value.(string)), nil+ case "user_type":+ v1UserType, ok := v1pb.UserType_value[value.(string)]+ if !ok {+ return nil, errors.Errorf("invalid user type filter %q", value)+ }+ principalType, err := convertToPrincipalType(v1pb.UserType(v1UserType))+ if err != nil {+ return nil, errors.Errorf("failed to parse the user type %q with error: %v", v1UserType, err)+ }+ return qb.Q().Space("principal.type = ?", principalType.String()), nil+ case "state":+ stateStr, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("state value must be string, got %T", value)+ }+ // Try with STATE_ prefix first (e.g., "STATE_ACTIVE", "STATE_DELETED")+ v1State, ok := v1pb.State_value[stateStr]+ if !ok {+ // If not found, try without STATE_ prefix (e.g., "ACTIVE", "DELETED")+ if v, exists := v1pb.State_value[strings.TrimPrefix(stateStr, "STATE_")]; exists {+ v1State = v+ ok = true+ }+ }+ if !ok {+ return nil, errors.Errorf("invalid state filter %q", value)+ }+ return qb.Q().Space("principal.deleted = ?", v1pb.State(v1State) == v1pb.State_DELETED), nil+ case "project":+ pid, err := common.GetProjectID(value.(string))+ if err != nil {+ return nil, errors.Errorf("invalid project filter %q", value)+ }+ projectID = &pid+ return qb.Q().Space("TRUE"), nil+ default:+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ }++ parseToUserTypeSQL := func(expr celast.Expr, relation string) (*qb.Query, error) {+ variable, value := getVariableAndValueFromExpr(expr)+ if variable != "user_type" {+ return nil, errors.Errorf(`only "user_type" support "user_type in [xx]"/"!(user_type in [xx])" operator`)+ }++ rawTypeList, ok := value.([]any)+ if !ok {+ return nil, errors.Errorf("invalid user_type value %q", value)+ }+ if len(rawTypeList) == 0 {+ return nil, errors.Errorf("empty user_type filter")+ }++ userTypeList := make([]any, 0, len(rawTypeList))+ for _, rawType := range rawTypeList {+ v1UserType, ok := v1pb.UserType_value[rawType.(string)]+ if !ok {+ return nil, errors.Errorf("invalid user type filter %q", rawType)+ }+ principalType, err := convertToPrincipalType(v1pb.UserType(v1UserType))+ if err != nil {+ return nil, errors.Errorf("failed to parse the user type %q with error: %v", v1UserType, err)+ }+ userTypeList = append(userTypeList, principalType.String())+ }++ placeholders := strings.Repeat("?,", len(userTypeList))+ placeholders = placeholders[:len(placeholders)-1]+ return qb.Q().Space(fmt.Sprintf("principal.type %s (%s)", relation, placeholders), userTypeList...), nil+ }++ getFilter = func(expr celast.Expr) (*qb.Query, error) {+ q := qb.Q()+ switch expr.Kind() {+ case celast.CallKind:+ functionName := expr.AsCall().FunctionName()+ switch functionName {+ case celoperators.LogicalOr:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.Or("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.LogicalAnd:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.And("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.Equals:+ variable, value := getVariableAndValueFromExpr(expr)+ return parseToSQL(variable, value)+ case celoverloads.Matches:+ variable := expr.AsCall().Target().AsIdent()+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`invalid args for %q`, variable)+ }+ value := args[0].AsLiteral().Value()+ if variable != "name" && variable != "email" {+ return nil, errors.Errorf(`only "name" and "email" support %q operator, but found %q`, celoverloads.Matches, variable)+ }+ strValue, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)+ }+ return qb.Q().Space("LOWER(principal."+variable+") LIKE ?", "%"+strings.ToLower(strValue)+"%"), nil+ case celoperators.In:+ return parseToUserTypeSQL(expr, "IN")+ case celoperators.LogicalNot:+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`only support !(user_type in ["{type1}", "{type2}"]) format`)+ }+ return parseToUserTypeSQL(args[0], "NOT IN")+ default:+ return nil, errors.Errorf("unexpected function %v", functionName)+ }+ default:+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())+ }+ }++ q, err := getFilter(ast.NativeRep().Expr())+ if err != nil {+ return nil, err+ }++ return &GetListUserFilterResult{+ Query: qb.Q().Space("(?)", q),+ ProjectID: projectID,+ }, nil+}diff --git a/backend/store/principal_filter_test.go b/backend/store/principal_filter_test.gonew file mode 100644index 0000000000..ab6a2b5604--- /dev/null+++ b/backend/store/principal_filter_test.go@@ -0,0 +1,187 @@+package store++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++func TestGetListUserFilter(t *testing.T) {+ tests := []struct {+ name string+ filter string+ wantSQL string+ wantArgs []any+ wantProject *string+ wantErr bool+ errContains string+ }{+ {+ name: "empty filter",+ filter: "",+ wantSQL: "",+ wantArgs: nil,+ wantErr: false,+ },+ {+ name: "name filter",+ filter: `name == "ed"`,+ wantSQL: "(principal.name = $1)",+ wantArgs: []any{"ed"},+ wantErr: false,+ },+ {+ name: "email filter",+ filter: `email == "[email protected]"`,+ wantSQL: "(principal.email = $1)",+ wantArgs: []any{"[email protected]"},+ wantErr: false,+ },+ {+ name: "name matches",+ filter: `name.matches("ED")`,+ wantSQL: "(LOWER(principal.name) LIKE $1)",+ wantArgs: []any{"%ed%"},+ wantErr: false,+ },+ {+ name: "email matches",+ filter: `email.matches("test")`,+ wantSQL: "(LOWER(principal.email) LIKE $1)",+ wantArgs: []any{"%test%"},+ wantErr: false,+ },+ {+ name: "user_type filter - SERVICE_ACCOUNT",+ filter: `user_type == "SERVICE_ACCOUNT"`,+ wantSQL: "(principal.type = $1)",+ wantArgs: []any{storepb.PrincipalType_SERVICE_ACCOUNT.String()},+ wantErr: false,+ },+ {+ name: "user_type filter - USER",+ filter: `user_type == "USER"`,+ wantSQL: "(principal.type = $1)",+ wantArgs: []any{storepb.PrincipalType_END_USER.String()},+ wantErr: false,+ },+ {+ name: "user_type in list",+ filter: `user_type in ["SERVICE_ACCOUNT", "USER"]`,+ wantSQL: "(principal.type IN ($1,$2))",+ wantArgs: []any{storepb.PrincipalType_SERVICE_ACCOUNT.String(), storepb.PrincipalType_END_USER.String()},+ wantErr: false,+ },+ {+ name: "user_type not in list",+ filter: `!(user_type in ["SERVICE_ACCOUNT"])`,+ wantSQL: "(principal.type NOT IN ($1))",+ wantArgs: []any{storepb.PrincipalType_SERVICE_ACCOUNT.String()},+ wantErr: false,+ },+ {+ name: "state filter - STATE_ACTIVE",+ filter: `state == "STATE_ACTIVE"`,+ wantSQL: "(principal.deleted = $1)",+ wantArgs: []any{false},+ wantErr: false,+ },+ {+ name: "state filter - STATE_DELETED",+ filter: `state == "STATE_DELETED"`,+ wantSQL: "(principal.deleted = $1)",+ wantArgs: []any{true},+ wantErr: false,+ },+ {+ name: "project filter",+ filter: `project == "projects/sample-project"`,+ wantSQL: "(TRUE)",+ wantArgs: []any{},+ wantProject: func() *string { s := "sample-project"; return &s }(),+ wantErr: false,+ },+ {+ name: "AND condition",+ filter: `name == "ed" && email == "[email protected]"`,+ wantSQL: "((principal.name = $1 AND principal.email = $2))",+ wantArgs: []any{"ed", "[email protected]"},+ wantErr: false,+ },+ {+ name: "OR condition",+ filter: `name == "ed" || name == "alice"`,+ wantSQL: "((principal.name = $1 OR principal.name = $2))",+ wantArgs: []any{"ed", "alice"},+ wantErr: false,+ },+ {+ name: "complex nested condition",+ filter: `(name == "ed" || name == "alice") && user_type == "USER"`,+ wantSQL: "(((principal.name = $1 OR principal.name = $2) AND principal.type = $3))",+ wantArgs: []any{"ed", "alice", storepb.PrincipalType_END_USER.String()},+ wantErr: false,+ },+ {+ name: "unsupported variable",+ filter: `title == "ed"`,+ wantErr: true,+ errContains: "unsupport variable",+ },+ {+ name: "invalid filter syntax",+ filter: `invalid syntax {{`,+ wantErr: true,+ errContains: "failed to parse filter",+ },+ {+ name: "invalid user type",+ filter: `user_type == "INVALID_TYPE"`,+ wantErr: true,+ errContains: "invalid user type filter",+ },+ {+ name: "invalid state",+ filter: `state == "INVALID_STATE"`,+ wantErr: true,+ errContains: "invalid state filter",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ result, err := GetListUserFilter(tt.filter)++ if tt.wantErr {+ require.Error(t, err)+ if tt.errContains != "" {+ require.ErrorContains(t, err, tt.errContains)+ }+ return+ }++ require.NoError(t, err)++ if tt.filter == "" {+ require.Nil(t, result)+ return+ }++ require.NotNil(t, result)++ if tt.wantProject != nil {+ require.NotNil(t, result.ProjectID)+ require.Equal(t, *tt.wantProject, *result.ProjectID)+ } else {+ require.Nil(t, result.ProjectID)+ }++ sql, args, err := result.Query.ToSQL()+ require.NoError(t, err)+ require.Equal(t, tt.wantSQL, sql)+ require.Equal(t, tt.wantArgs, args)+ })+ }+}