Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(qb): plan filter (#17880)
p0nyf2d2d4b67600
diff --git a/backend/api/v1/plan_service.go b/backend/api/v1/plan_service.goindex 80f20bed69..16ee9aa031 100644--- a/backend/api/v1/plan_service.go+++ b/backend/api/v1/plan_service.go@@ -2,18 +2,15 @@ package v1import ("context"- "fmt""log/slog""maps""slices""strings"- "time""connectrpc.com/connect""github.com/google/cel-go/cel"celast "github.com/google/cel-go/common/ast"celoperators "github.com/google/cel-go/common/operators"- celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors""github.com/bytebase/bytebase/backend/common"@@ -170,9 +167,11 @@ func (s *PlanService) SearchPlans(ctx context.Context, request *connect.Request[if projectID != "-" {find.ProjectID = &projectID}- if err := s.buildPlanFindWithFilter(ctx, find, req.Filter); err != nil {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to build plan find with filter, error: %v", err))+ filterQ, err := s.store.GetListPlanFilter(ctx, req.Filter)+ if err != nil {+ return nil, connect.NewError(connect.CodeInvalidArgument, err)}+ find.FilterQ = filterQplans, err := s.store.ListPlans(ctx, find)if err != nil {@@ -1097,161 +1096,6 @@ func (s *PlanService) BatchCancelPlanCheckRuns(ctx context.Context, request *conreturn connect.NewResponse(&v1pb.BatchCancelPlanCheckRunsResponse{}), nil}-func (s *PlanService) buildPlanFindWithFilter(ctx context.Context, planFind *store.FindPlanMessage, filter string) error {- if filter == "" {- return nil- }-- e, err := cel.NewEnv()- if err != nil {- return connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))- }- ast, iss := e.Parse(filter)- if iss != nil {- return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))- }-- var getFilter func(expr celast.Expr) (string, error)- var positionalArgs []any-- getFilter = func(expr celast.Expr) (string, error) {- switch expr.Kind() {- case celast.CallKind:- functionName := expr.AsCall().FunctionName()- switch functionName {- case celoperators.LogicalAnd:- return getSubConditionFromExpr(expr, getFilter, "AND")- case celoperators.Equals:- variable, value := getVariableAndValueFromExpr(expr)- switch variable {- case "creator":- user, err := s.getUserByIdentifier(ctx, value.(string))- if err != nil {- return "", connect.NewError(connect.CodeInternal, errors.Errorf("failed to get user %v with error %v", value, err.Error()))- }- positionalArgs = append(positionalArgs, user.ID)- return fmt.Sprintf("plan.creator_id = $%d", len(positionalArgs)), nil- case "has_pipeline":- hasPipeline, ok := value.(bool)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`"has_pipeline" should be bool`))- }- if !hasPipeline {- return "plan.pipeline_id IS NULL", nil- }- return "plan.pipeline_id IS NOT NULL", nil- case "has_issue":- hasIssue, ok := value.(bool)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`"has_issue" should be bool`))- }- if !hasIssue {- return "issue.id IS NULL", nil- }- return "issue.id IS NOT NULL", nil- case "title":- positionalArgs = append(positionalArgs, value)- return fmt.Sprintf("plan.name = $%d", len(positionalArgs)), nil- case "spec_type":- specType, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("spec_type value must be a string"))- }- switch specType {- case "create_database_config":- return "EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'createDatabaseConfig' IS NOT NULL)", nil- case "change_database_config":- return "EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'changeDatabaseConfig' IS NOT NULL)", nil- case "export_data_config":- return "EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'exportDataConfig' IS NOT NULL)", nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid spec_type value: %s, must be one of: create_database_config, change_database_config, export_data_config", specType))- }- case "state":- v1State, ok := v1pb.State_value[value.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid state filter %q", value))- }- positionalArgs = append(positionalArgs, v1pb.State(v1State) == v1pb.State_DELETED)- return fmt.Sprintf("plan.deleted = $%d", len(positionalArgs)), nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported variable %q", variable))- }- case celoperators.GreaterEquals, celoperators.LessEquals:- variable, rawValue := getVariableAndValueFromExpr(expr)- value, ok := rawValue.(string)- if !ok {- return "", errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue)- }- if variable != "create_time" {- return "", errors.Errorf(`">=" and "<=" are only supported for "create_time"`)- }- t, err := time.Parse(time.RFC3339, value)- if err != nil {- return "", errors.Errorf("failed to parse time %v, error: %v", value, err)- }- positionalArgs = append(positionalArgs, t)- if functionName == celoperators.GreaterEquals {- return fmt.Sprintf("plan.created_at >= $%d", len(positionalArgs)), nil- }- return fmt.Sprintf("plan.created_at <= $%d", len(positionalArgs)), nil- case celoverloads.Matches:- variable := expr.AsCall().Target().AsIdent()- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))- }- value := args[0].AsLiteral().Value()- strValue, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))- }- strValue = strings.ToLower(strValue)-- switch variable {- case "title":- return "LOWER(plan.name) LIKE '%" + strValue + "%'", nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "title" supports %q operator, but found %q`, celoverloads.Matches, variable))- }- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported function %v", functionName))- }- default:- return "", errors.Errorf("unexpected expr kind %v", expr.Kind())- }- }-- where, err := getFilter(ast.NativeRep().Expr())- if err != nil {- return err- }-- if where != "" {- planFind.Filter = &store.ListResourceFilter{- Where: "(" + where + ")",- Args: positionalArgs,- }- }-- return nil-}--func (s *PlanService) getUserByIdentifier(ctx context.Context, identifier string) (*store.UserMessage, error) {- email := strings.TrimPrefix(identifier, "users/")- if email == "" {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("invalid empty creator identifier"))- }- user, err := s.store.GetUserByEmail(ctx, email)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf(`failed to find user "%s" with error: %v`, email, err))- }- if user == nil {- return nil, errors.Errorf("cannot found user %s", email)- }- return user, nil-}-// diffSpecs checks if there are any specs removed, added or updated in the new plan.// It performs a deep comparison of all spec fields using protocol buffer comparison.// Returns (removed, added, updated) slices of specs.diff --git a/backend/store/plan.go b/backend/store/plan.goindex 9845adb78c..947094092e 100644--- a/backend/store/plan.go+++ b/backend/store/plan.go@@ -8,6 +8,10 @@ import ("strings""time"+ "github.com/google/cel-go/cel"+ celast "github.com/google/cel-go/common/ast"+ celoperators "github.com/google/cel-go/common/operators"+ celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson"@@ -15,6 +19,7 @@ import ("github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/common/qb"storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")// PlanMessage is the message for plan.@@ -42,7 +47,7 @@ type FindPlanMessage struct {Limit *intOffset *int- Filter *ListResourceFilter+ FilterQ *qb.Query}// UpdatePlanMessage is the message to update a plan.@@ -142,9 +147,8 @@ func (s *Store) ListPlans(ctx context.Context, find *FindPlanMessage) ([]*PlanMeWHERE TRUE`)- if filter := find.Filter; filter != nil {- // Convert $1, $2, etc. to ? for qb- q.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)+ if filterQ := find.FilterQ; filterQ != nil {+ q.And("?", filterQ)}if v := find.UID; v != nil {q.And("plan.id = ?", *v)@@ -289,3 +293,164 @@ func (s *Store) UpdatePlan(ctx context.Context, patch *UpdatePlanMessage) errorreturn nil}++// GetListPlanFilter parses a CEL filter expression into a query builder query for listing plans.+func (s *Store) GetListPlanFilter(ctx context.Context, filter string) (*qb.Query, error) {+ if filter == "" {+ return nil, nil+ }++ e, err := cel.NewEnv()+ if err != nil {+ return nil, errors.Errorf("failed to create cel env")+ }+ ast, iss := e.Parse(filter)+ if iss != nil {+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())+ }++ var getFilter func(expr celast.Expr) (*qb.Query, error)++ getFilter = func(expr celast.Expr) (*qb.Query, error) {+ q := qb.Q()+ switch expr.Kind() {+ case celast.CallKind:+ functionName := expr.AsCall().FunctionName()+ switch functionName {+ case celoperators.LogicalAnd:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.And("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.Equals:+ variable, value := getVariableAndValueFromExpr(expr)+ switch variable {+ case "creator":+ user, err := s.getUserByIdentifier(ctx, value.(string))+ if err != nil {+ return nil, errors.Errorf("failed to get user %v with error %v", value, err.Error())+ }+ return qb.Q().Space("plan.creator_id = ?", user.ID), nil+ case "has_pipeline":+ hasPipeline, ok := value.(bool)+ if !ok {+ return nil, errors.Errorf(`"has_pipeline" should be bool`)+ }+ if !hasPipeline {+ return qb.Q().Space("plan.pipeline_id IS NULL"), nil+ }+ return qb.Q().Space("plan.pipeline_id IS NOT NULL"), nil+ case "has_issue":+ hasIssue, ok := value.(bool)+ if !ok {+ return nil, errors.Errorf(`"has_issue" should be bool`)+ }+ if !hasIssue {+ return qb.Q().Space("issue.id IS NULL"), nil+ }+ return qb.Q().Space("issue.id IS NOT NULL"), nil+ case "title":+ return qb.Q().Space("plan.name = ?", value), nil+ case "spec_type":+ specType, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("spec_type value must be a string")+ }+ switch specType {+ case "create_database_config":+ return qb.Q().Space("EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'createDatabaseConfig' IS NOT NULL)"), nil+ case "change_database_config":+ return qb.Q().Space("EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'changeDatabaseConfig' IS NOT NULL)"), nil+ case "export_data_config":+ return qb.Q().Space("EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'exportDataConfig' IS NOT NULL)"), nil+ default:+ return nil, errors.Errorf("invalid spec_type value: %s, must be one of: create_database_config, change_database_config, export_data_config", specType)+ }+ case "state":+ stateStr, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("state value must be string, got %T", value)+ }+ v1State, ok := v1pb.State_value[stateStr]+ if !ok {+ if v, exists := v1pb.State_value[strings.TrimPrefix(stateStr, "STATE_")]; exists {+ v1State = v+ ok = true+ }+ }+ if !ok {+ return nil, errors.Errorf("invalid state filter %q", value)+ }+ return qb.Q().Space("plan.deleted = ?", v1pb.State(v1State) == v1pb.State_DELETED), nil+ default:+ return nil, errors.Errorf("unsupported variable %q", variable)+ }+ case celoperators.GreaterEquals, celoperators.LessEquals:+ variable, rawValue := getVariableAndValueFromExpr(expr)+ value, ok := rawValue.(string)+ if !ok {+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue)+ }+ if variable != "create_time" {+ return nil, errors.Errorf(`">=" and "<=" are only supported for "create_time"`)+ }+ t, err := time.Parse(time.RFC3339, value)+ if err != nil {+ return nil, errors.Errorf("failed to parse time %v, error: %v", value, err)+ }+ if functionName == celoperators.GreaterEquals {+ return qb.Q().Space("plan.created_at >= ?", t), nil+ }+ return qb.Q().Space("plan.created_at <= ?", t), nil+ case celoverloads.Matches:+ variable := expr.AsCall().Target().AsIdent()+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`invalid args for %q`, variable)+ }+ value := args[0].AsLiteral().Value()+ strValue, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)+ }+ strValue = strings.ToLower(strValue)++ switch variable {+ case "title":+ return qb.Q().Space("LOWER(plan.name) LIKE ?", "%"+strValue+"%"), nil+ default:+ return nil, errors.Errorf(`only "title" supports %q operator, but found %q`, celoverloads.Matches, variable)+ }+ default:+ return nil, errors.Errorf("unsupported function %v", functionName)+ }+ default:+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())+ }+ }++ q, err := getFilter(ast.NativeRep().Expr())+ if err != nil {+ return nil, err+ }+ return qb.Q().Space("(?)", q), nil+}++func (s *Store) getUserByIdentifier(ctx context.Context, identifier string) (*UserMessage, error) {+ email := strings.TrimPrefix(identifier, "users/")+ if email == "" {+ return nil, errors.New("invalid empty creator identifier")+ }+ user, err := s.GetUserByEmail(ctx, email)+ if err != nil {+ return nil, errors.Errorf(`failed to find user "%s" with error: %v`, email, err)+ }+ if user == nil {+ return nil, errors.Errorf("cannot found user %s", email)+ }+ return user, nil+}diff --git a/backend/store/plan_filter_test.go b/backend/store/plan_filter_test.gonew file mode 100644index 0000000000..11b9a04669--- /dev/null+++ b/backend/store/plan_filter_test.go@@ -0,0 +1,232 @@+package store++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++func TestGetListPlanFilter(t *testing.T) {+ // Create a minimal store instance for testing+ // Note: Some tests will be limited without a full database connection+ s := &Store{}+ ctx := context.Background()++ tests := []struct {+ name string+ filter string+ wantSQL string+ wantArgs []any+ wantErr bool+ errContains string+ skipTest bool // Skip tests that require database access+ }{+ {+ name: "empty filter",+ filter: "",+ wantSQL: "",+ wantArgs: nil,+ wantErr: false,+ },+ {+ name: "has_pipeline filter - true",+ filter: `has_pipeline == true`,+ wantSQL: "(plan.pipeline_id IS NOT NULL)",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "has_pipeline filter - false",+ filter: `has_pipeline == false`,+ wantSQL: "(plan.pipeline_id IS NULL)",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "has_issue filter - true",+ filter: `has_issue == true`,+ wantSQL: "(issue.id IS NOT NULL)",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "has_issue filter - false",+ filter: `has_issue == false`,+ wantSQL: "(issue.id IS NULL)",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "title filter",+ filter: `title == "My Plan"`,+ wantSQL: "(plan.name = $1)",+ wantArgs: []any{"My Plan"},+ wantErr: false,+ },+ {+ name: "title matches",+ filter: `title.matches("test")`,+ wantSQL: "(LOWER(plan.name) LIKE $1)",+ wantArgs: []any{"%test%"},+ wantErr: false,+ },+ {+ name: "spec_type filter - create_database_config",+ filter: `spec_type == "create_database_config"`,+ wantSQL: "(EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'createDatabaseConfig' IS NOT NULL))",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "spec_type filter - change_database_config",+ filter: `spec_type == "change_database_config"`,+ wantSQL: "(EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'changeDatabaseConfig' IS NOT NULL))",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "spec_type filter - export_data_config",+ filter: `spec_type == "export_data_config"`,+ wantSQL: "(EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'exportDataConfig' IS NOT NULL))",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "state filter - ACTIVE",+ filter: `state == "STATE_ACTIVE"`,+ wantSQL: "(plan.deleted = $1)",+ wantArgs: []any{false},+ wantErr: false,+ },+ {+ name: "state filter - DELETED",+ filter: `state == "STATE_DELETED"`,+ wantSQL: "(plan.deleted = $1)",+ wantArgs: []any{true},+ wantErr: false,+ },+ {+ name: "create_time greater than or equal",+ filter: `create_time >= "2024-01-01T00:00:00Z"`,+ wantSQL: "(plan.created_at >= $1)",+ wantArgs: []any{func() time.Time {+ ts, _ := time.Parse(time.RFC3339, "2024-01-01T00:00:00Z")+ return ts+ }()},+ wantErr: false,+ },+ {+ name: "create_time less than or equal",+ filter: `create_time <= "2024-12-31T23:59:59Z"`,+ wantSQL: "(plan.created_at <= $1)",+ wantArgs: []any{func() time.Time {+ ts, _ := time.Parse(time.RFC3339, "2024-12-31T23:59:59Z")+ return ts+ }()},+ wantErr: false,+ },+ {+ name: "AND condition with has_pipeline and has_issue",+ filter: `has_pipeline == true && has_issue == true`,+ wantSQL: "((plan.pipeline_id IS NOT NULL AND issue.id IS NOT NULL))",+ wantArgs: []any{},+ wantErr: false,+ },+ {+ name: "complex AND condition",+ filter: `title == "Test Plan" && state == "STATE_ACTIVE" && has_pipeline == true`,+ wantSQL: "(((plan.name = $1 AND plan.deleted = $2) AND plan.pipeline_id IS NOT NULL))",+ wantArgs: []any{"Test Plan", false},+ wantErr: false,+ },+ {+ name: "creator filter requires database",+ filter: `creator == "users/[email protected]"`,+ skipTest: true,+ wantErr: false, // Would work with database+ errContains: "",+ },+ {+ name: "invalid filter syntax",+ filter: `invalid syntax {{`,+ wantErr: true,+ errContains: "failed to parse filter",+ },+ {+ name: "unsupported variable",+ filter: `unsupported == "value"`,+ wantErr: true,+ errContains: "unsupported variable",+ },+ {+ name: "invalid state value",+ filter: `state == "INVALID_STATE"`,+ wantErr: true,+ errContains: "invalid state filter",+ },+ {+ name: "invalid spec_type value",+ filter: `spec_type == "invalid_type"`,+ wantErr: true,+ errContains: "invalid spec_type value",+ },+ {+ name: "has_pipeline with non-bool value",+ filter: `has_pipeline == "true"`,+ wantErr: true,+ errContains: `"has_pipeline" should be bool`,+ },+ {+ name: "has_issue with non-bool value",+ filter: `has_issue == "true"`,+ wantErr: true,+ errContains: `"has_issue" should be bool`,+ },+ {+ name: "invalid time format",+ filter: `create_time >= "invalid-time"`,+ wantErr: true,+ errContains: "failed to parse time",+ },+ {+ name: "comparison operator on unsupported field",+ filter: `title >= "test"`,+ wantErr: true,+ errContains: `">=" and "<=" are only supported for "create_time"`,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if tt.skipTest {+ t.Skip("Test requires database connection")+ }++ q, err := s.GetListPlanFilter(ctx, tt.filter)++ if tt.wantErr {+ require.Error(t, err)+ if tt.errContains != "" {+ require.ErrorContains(t, err, tt.errContains)+ }+ return+ }++ require.NoError(t, err)++ if tt.filter == "" {+ require.Nil(t, q)+ return+ }++ require.NotNil(t, q)++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, tt.wantSQL, sql)+ require.Equal(t, tt.wantArgs, args)+ })+ }+}