Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(qb): project service filter (#17834)
p0nyd0aa4b4ad1b7
diff --git a/backend/api/v1/project_service.go b/backend/api/v1/project_service.goindex 1ed1a4d7f9..ed2b8194aa 100644--- a/backend/api/v1/project_service.go+++ b/backend/api/v1/project_service.go@@ -11,8 +11,6 @@ import ("connectrpc.com/connect""github.com/google/cel-go/cel"celast "github.com/google/cel-go/common/ast"- celoperators "github.com/google/cel-go/common/operators"- celoverloads "github.com/google/cel-go/common/overloads""github.com/gosimple/slug""github.com/pkg/errors""google.golang.org/genproto/googleapis/type/expr"@@ -83,11 +81,11 @@ func (s *ProjectService) ListProjects(ctx context.Context, req *connect.Request[Limit: &limitPlusOne,Offset: &offset.offset,}- filter, err := getListProjectFilter(req.Msg.Filter)+ filterQ, err := store.GetListProjectFilter(req.Msg.Filter)if err != nil {- return nil, err+ return nil, connect.NewError(connect.CodeInvalidArgument, err)}- find.Filter = filter+ find.FilterQ = filterQprojects, err := s.store.ListProjectV2(ctx, find)if err != nil {return nil, connect.NewError(connect.CodeInternal, err)@@ -110,113 +108,6 @@ func (s *ProjectService) ListProjects(ctx context.Context, req *connect.Request[return connect.NewResponse(response), nil}-func getListProjectFilter(filter string) (*store.ListResourceFilter, error) {- if filter == "" {- return nil, nil- }- e, err := cel.NewEnv()- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))- }- ast, iss := e.Parse(filter)- if iss != nil {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))- }-- var getFilter func(expr celast.Expr) (string, error)- var positionalArgs []any-- parseToSQL := func(variable, value any) (string, error) {- switch variable {- case "name":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("project.name = $%d", len(positionalArgs)), nil- case "resource_id":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("project.resource_id = $%d", len(positionalArgs)), nil- case "exclude_default":- if excludeDefault, ok := value.(bool); excludeDefault && ok {- positionalArgs = append(positionalArgs, common.DefaultProjectID)- return fmt.Sprintf("project.resource_id != $%d", len(positionalArgs)), nil- }- return "TRUE", nil- case "state":- v1State, ok := v1pb.State_value[value.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid state filter %q", value))- }- positionalArgs = append(positionalArgs, v1pb.State(v1State) == v1pb.State_DELETED)- return fmt.Sprintf("project.deleted = $%d", len(positionalArgs)), nil- default:- // Check if it's a label filter (e.g., "labels.environment" == "production")- varStr, ok := variable.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- if labelKey, ok := strings.CutPrefix(varStr, "labels."); ok {- return parseToLabelFilterSQL("project.setting", labelKey, value)- }- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- }-- getFilter = func(expr celast.Expr) (string, error) {- switch expr.Kind() {- case celast.CallKind:- functionName := expr.AsCall().FunctionName()- switch functionName {- case celoperators.LogicalOr:- return getSubConditionFromExpr(expr, getFilter, "OR")- case celoperators.LogicalAnd:- return getSubConditionFromExpr(expr, getFilter, "AND")- case celoperators.Equals:- variable, value := getVariableAndValueFromExpr(expr)- return parseToSQL(variable, value)- case celoverloads.Matches:- variable := expr.AsCall().Target().AsIdent()- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))- }- value := args[0].AsLiteral().Value()- strValue, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))- }-- switch variable {- case "name":- return "LOWER(project.name) LIKE '%" + strings.ToLower(strValue) + "%'", nil- case "resource_id":- return "LOWER(project.resource_id) LIKE '%" + strings.ToLower(strValue) + "%'", nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- case celoperators.In:- variable, value := getVariableAndValueFromExpr(expr)- if labelKey, ok := strings.CutPrefix(variable, "labels."); ok {- return parseToLabelFilterSQL("project.setting", labelKey, value)- }- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected %v operator for %v", functionName, variable))- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))- }- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected expr kind %v", expr.Kind()))- }- }-- where, err := getFilter(ast.NativeRep().Expr())- if err != nil {- return nil, err- }-- return &store.ListResourceFilter{- Args: positionalArgs,- Where: "(" + where + ")",- }, nil-}-// SearchProjects searches all projects on which the user has bb.projects.get permission.func (s *ProjectService) SearchProjects(ctx context.Context, req *connect.Request[v1pb.SearchProjectsRequest]) (*connect.Response[v1pb.SearchProjectsResponse], error) {user, ok := GetUserFromContext(ctx)@@ -239,11 +130,11 @@ func (s *ProjectService) SearchProjects(ctx context.Context, req *connect.RequesLimit: &limitPlusOne,Offset: &offset.offset,}- filter, err := getListProjectFilter(req.Msg.Filter)+ filterQ, err := store.GetListProjectFilter(req.Msg.Filter)if err != nil {- return nil, err+ return nil, connect.NewError(connect.CodeInvalidArgument, err)}- find.Filter = filter+ find.FilterQ = filterQprojects, err := s.store.ListProjectV2(ctx, find)if err != nil {diff --git a/backend/api/v1/project_service_test.go b/backend/api/v1/project_service_test.goindex 172a63ee89..ec8e152903 100644--- a/backend/api/v1/project_service_test.go+++ b/backend/api/v1/project_service_test.go@@ -6,16 +6,13 @@ import ("testing""time"- "connectrpc.com/connect""github.com/google/go-cmp/cmp"- "github.com/pkg/errors""github.com/stretchr/testify/assert""github.com/stretchr/testify/require""google.golang.org/genproto/googleapis/type/expr""google.golang.org/protobuf/testing/protocmp""google.golang.org/protobuf/types/known/durationpb"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/store"@@ -373,73 +370,6 @@ func TestFindIamPolicyDeltas(t *testing.T) {}}-func TestListProjectFilter(t *testing.T) {- testCases := []struct {- input string- want *store.ListResourceFilter- error *connect.Error- }{- {- input: `title == "sample project"`,- error: connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", "title")),- },- {- input: `name == "sample project"`,- want: &store.ListResourceFilter{- Where: `(project.name = $1)`,- Args: []any{"sample project"},- },- },- {- input: `name.matches("Sample")`,- want: &store.ListResourceFilter{- Where: `(LOWER(project.name) LIKE '%sample%')`,- },- },- {- input: `resource_id == "sample-project"`,- want: &store.ListResourceFilter{- Where: `(project.resource_id = $1)`,- Args: []any{"sample-project"},- },- },- {- input: `resource_id.matches("sample")`,- want: &store.ListResourceFilter{- Where: `(LOWER(project.resource_id) LIKE '%sample%')`,- },- },- {- input: `exclude_default == true`,- want: &store.ListResourceFilter{- Where: `(project.resource_id != $1)`,- Args: []any{common.DefaultProjectID},- },- },- {- input: `(name.matches("sample") || resource_id.matches("Sample")) && exclude_default == true`,- want: &store.ListResourceFilter{- Where: `(((LOWER(project.name) LIKE '%sample%') OR (LOWER(project.resource_id) LIKE '%sample%')) AND (project.resource_id != $1))`,- Args: []any{common.DefaultProjectID},- },- },- }-- for _, tc := range testCases {- filter, err := getListProjectFilter(tc.input)- if tc.error != nil {- require.Error(t, err)- connectErr := new(connect.Error)- require.True(t, errors.As(err, &connectErr))- require.Equal(t, tc.error.Message(), connectErr.Message())- require.Equal(t, tc.error.Code(), connectErr.Code())- } else {- require.NoError(t, err)- require.Equal(t, tc.want.Where, filter.Where)- }- }-}-func TestValidateLabels(t *testing.T) {tests := []struct {name stringdiff --git a/backend/store/project.go b/backend/store/project.goindex d3d5419811..bde11c1e52 100644--- a/backend/store/project.go+++ b/backend/store/project.go@@ -4,7 +4,12 @@ import ("context""database/sql""fmt"+ "strings"+ "github.com/google/cel-go/cel"+ celast "github.com/google/cel-go/common/ast"+ celoperators "github.com/google/cel-go/common/operators"+ celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors""google.golang.org/genproto/googleapis/type/expr""google.golang.org/protobuf/encoding/protojson"@@ -12,6 +17,7 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/qb"storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")// ProjectMessage is the message for project.@@ -34,7 +40,7 @@ type FindProjectMessage struct {ShowDeleted boolLimit *intOffset *int- Filter *ListResourceFilter+ FilterQ *qb.Query}// UpdateProjectMessage is the message for updating a project.@@ -279,8 +285,8 @@ func updateProjectImplV2(ctx context.Context, txn *sql.Tx, patch *UpdateProjectMfunc (s *Store) listProjectImplV2(ctx context.Context, txn *sql.Tx, find *FindProjectMessage) ([]*ProjectMessage, error) {q := qb.Q().Space("SELECT resource_id, name, data_classification_config_id, setting, deleted FROM project WHERE TRUE")- if filter := find.Filter; filter != nil {- q.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)+ if filterQ := find.FilterQ; filterQ != nil {+ q.And("?", filterQ)}if v := find.ResourceID; v != nil {q.And("resource_id = ?", *v)@@ -573,3 +579,153 @@ func (s *Store) DeleteProject(ctx context.Context, resourceID string) error {return nil}++func GetListProjectFilter(filter string) (*qb.Query, error) {+ if filter == "" {+ return nil, nil+ }++ e, err := cel.NewEnv()+ if err != nil {+ return nil, errors.Errorf("failed to create cel env")+ }+ ast, iss := e.Parse(filter)+ if iss != nil {+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())+ }++ var getFilter func(expr celast.Expr) (*qb.Query, error)++ parseToLabelFilterSQL := func(resource, key string, value any) (*qb.Query, error) {+ switch v := value.(type) {+ case string:+ return qb.Q().Space(fmt.Sprintf("%s->'labels'->>'%s' = ?", resource, key), v), nil+ case []any:+ if len(v) == 0 {+ return nil, errors.Errorf("empty label filter")+ }++ labelValueList := make([]any, len(v))+ for i, raw := range v {+ str, ok := raw.(string)+ if !ok {+ return nil, errors.Errorf("label value must be string, got %T", raw)+ }+ labelValueList[i] = str+ }+ placeholders := strings.Repeat("?,", len(labelValueList))+ placeholders = placeholders[:len(placeholders)-1]+ return qb.Q().Space(fmt.Sprintf("%s->'labels'->>'%s' IN (%s)", resource, key, placeholders), labelValueList...), nil+ default:+ return nil, errors.Errorf("empty value %v for label filter", value)+ }+ }++ parseToSQL := func(variable, value any) (*qb.Query, error) {+ switch variable {+ case "name":+ return qb.Q().Space("project.name = ?", value.(string)), nil+ case "resource_id":+ return qb.Q().Space("project.resource_id = ?", value.(string)), nil+ case "exclude_default":+ if excludeDefault, ok := value.(bool); excludeDefault && ok {+ return qb.Q().Space("project.resource_id != ?", common.DefaultProjectID), nil+ }+ return qb.Q().Space("TRUE"), nil+ case "state":+ stateStr, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("state value must be string, got %T", value)+ }+ // Try with STATE_ prefix first (e.g., "STATE_ACTIVE", "STATE_DELETED")+ v1State, ok := v1pb.State_value[stateStr]+ if !ok {+ // If not found, try without STATE_ prefix (e.g., "ACTIVE", "DELETED")+ if v, exists := v1pb.State_value[strings.TrimPrefix(stateStr, "STATE_")]; exists {+ v1State = v+ ok = true+ }+ }+ if !ok {+ return nil, errors.Errorf("invalid state filter %q", value)+ }+ return qb.Q().Space("project.deleted = ?", v1pb.State(v1State) == v1pb.State_DELETED), nil+ default:+ varStr, ok := variable.(string)+ if !ok {+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ if labelKey, ok := strings.CutPrefix(varStr, "labels."); ok {+ return parseToLabelFilterSQL("project.setting", labelKey, value)+ }+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ }++ getFilter = func(expr celast.Expr) (*qb.Query, error) {+ q := qb.Q()+ switch expr.Kind() {+ case celast.CallKind:+ functionName := expr.AsCall().FunctionName()+ switch functionName {+ case celoperators.LogicalOr:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.Or("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.LogicalAnd:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.And("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.Equals:+ variable, value := getVariableAndValueFromExpr(expr)+ return parseToSQL(variable, value)+ case celoverloads.Matches:+ variable := expr.AsCall().Target().AsIdent()+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`invalid args for %q`, variable)+ }+ value := args[0].AsLiteral().Value()+ strValue, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)+ }++ switch variable {+ case "name":+ return qb.Q().Space("LOWER(project.name) LIKE ?", "%"+strings.ToLower(strValue)+"%"), nil+ case "resource_id":+ return qb.Q().Space("LOWER(project.resource_id) LIKE ?", "%"+strings.ToLower(strValue)+"%"), nil+ default:+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ case celoperators.In:+ variable, value := getVariableAndValueFromExpr(expr)+ if labelKey, ok := strings.CutPrefix(variable, "labels."); ok {+ return parseToLabelFilterSQL("project.setting", labelKey, value)+ }+ return nil, errors.Errorf("unexpected %v operator for %v", functionName, variable)+ default:+ return nil, errors.Errorf("unexpected function %v", functionName)+ }+ default:+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())+ }+ }++ q, err := getFilter(ast.NativeRep().Expr())+ if err != nil {+ return nil, err+ }+ return qb.Q().Space("(?)", q), nil+}diff --git a/backend/store/project_filter_test.go b/backend/store/project_filter_test.gonew file mode 100644index 0000000000..313288a2ec--- /dev/null+++ b/backend/store/project_filter_test.go@@ -0,0 +1,163 @@+package store++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestGetListProjectFilter(t *testing.T) {+ tests := []struct {+ name string+ filter string+ wantSQL string+ wantArgs []any+ wantErr bool+ errContains string+ }{+ {+ name: "empty filter",+ filter: "",+ wantSQL: "",+ wantArgs: nil,+ wantErr: false,+ },+ {+ name: "name filter",+ filter: `name == "test-project"`,+ wantSQL: "(project.name = $1)",+ wantArgs: []any{"test-project"},+ wantErr: false,+ },+ {+ name: "resource_id filter",+ filter: `resource_id == "test-project-id"`,+ wantSQL: "(project.resource_id = $1)",+ wantArgs: []any{"test-project-id"},+ wantErr: false,+ },+ {+ name: "exclude_default filter - true",+ filter: `exclude_default == true`,+ wantSQL: "(project.resource_id != $1)",+ wantArgs: []any{"default"},+ wantErr: false,+ },+ {+ name: "exclude_default filter - false",+ filter: `exclude_default == false`,+ wantSQL: "(TRUE)",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "state filter - ACTIVE",+ filter: `state == "STATE_ACTIVE"`,+ wantSQL: "(project.deleted = $1)",+ wantArgs: []any{false},+ wantErr: false,+ },+ {+ name: "state filter - DELETED",+ filter: `state == "STATE_DELETED"`,+ wantSQL: "(project.deleted = $1)",+ wantArgs: []any{true},+ wantErr: false,+ },+ {+ name: "name matches",+ filter: `name.matches("test")`,+ wantSQL: "(LOWER(project.name) LIKE $1)",+ wantArgs: []any{"%test%"},+ wantErr: false,+ },+ {+ name: "resource_id matches",+ filter: `resource_id.matches("prod")`,+ wantSQL: "(LOWER(project.resource_id) LIKE $1)",+ wantArgs: []any{"%prod%"},+ wantErr: false,+ },+ {+ name: "label filter - string",+ filter: `labels.environment == "production"`,+ wantSQL: "(project.setting->'labels'->>'environment' = $1)",+ wantArgs: []any{"production"},+ wantErr: false,+ },+ {+ name: "label filter - in operator",+ filter: `labels.environment in ["production", "staging"]`,+ wantSQL: "(project.setting->'labels'->>'environment' IN ($1,$2))",+ wantArgs: []any{"production", "staging"},+ wantErr: false,+ },+ {+ name: "AND condition with name and resource_id",+ filter: `name == "test-project" && resource_id == "test-id"`,+ wantSQL: "((project.name = $1 AND project.resource_id = $2))",+ wantArgs: []any{"test-project", "test-id"},+ wantErr: false,+ },+ {+ name: "OR condition with name",+ filter: `name == "project1" || name == "project2"`,+ wantSQL: "((project.name = $1 OR project.name = $2))",+ wantArgs: []any{"project1", "project2"},+ wantErr: false,+ },+ {+ name: "complex nested AND/OR",+ filter: `(name == "p1" || name == "p2") && exclude_default == true`,+ wantSQL: "(((project.name = $1 OR project.name = $2) AND project.resource_id != $3))",+ wantArgs: []any{"p1", "p2", "default"},+ wantErr: false,+ },+ {+ name: "invalid filter syntax",+ filter: `invalid syntax {{`,+ wantErr: true,+ errContains: "failed to parse filter",+ },+ {+ name: "unsupported variable",+ filter: `unsupported == "value"`,+ wantErr: true,+ errContains: "unsupport variable",+ },+ {+ name: "invalid state value",+ filter: `state == "INVALID_STATE"`,+ wantErr: true,+ errContains: "invalid state filter",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ q, err := GetListProjectFilter(tt.filter)++ if tt.wantErr {+ require.Error(t, err)+ if tt.errContains != "" {+ require.ErrorContains(t, err, tt.errContains)+ }+ return+ }++ require.NoError(t, err)++ if tt.filter == "" {+ require.Nil(t, q)+ return+ }++ require.NotNil(t, q)++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, tt.wantSQL, sql)+ require.Equal(t, tt.wantArgs, args)+ })+ }+}