Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(qb): pipeline (#17883)
p0ny9a9656627f95
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.goindex 0473a01d37..4184794c3c 100644--- a/backend/api/v1/rollout_service.go+++ b/backend/api/v1/rollout_service.go@@ -3,16 +3,12 @@ package v1import ("context""database/sql"- "fmt""log/slog""slices""strings""time""connectrpc.com/connect"- "github.com/google/cel-go/cel"- celast "github.com/google/cel-go/common/ast"- celoperators "github.com/google/cel-go/common/operators""github.com/jackc/pgtype""github.com/pkg/errors""google.golang.org/protobuf/types/known/timestamppb"@@ -206,123 +202,12 @@ func (s *RolloutService) buildRolloutFindWithFilter(ctx context.Context, pipelinreturn nil}- e, err := cel.NewEnv()- if err != nil {- return connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))- }- ast, iss := e.Parse(filter)- if iss != nil {- return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))- }-- var getFilter func(expr celast.Expr) (string, error)- var positionalArgs []any-- getFilter = func(expr celast.Expr) (string, error) {- switch expr.Kind() {- case celast.CallKind:- functionName := expr.AsCall().FunctionName()- switch functionName {- case celoperators.LogicalAnd:- return getSubConditionFromExpr(expr, getFilter, "AND")- case celoperators.Equals:- variable, value := getVariableAndValueFromExpr(expr)- switch variable {- case "creator":- user, err := s.getUserByIdentifier(ctx, value.(string))- if err != nil {- return "", connect.NewError(connect.CodeInternal, errors.Errorf("failed to get user %v with error %v", value, err.Error()))- }- positionalArgs = append(positionalArgs, user.ID)- return fmt.Sprintf("pipeline.creator_id = $%d", len(positionalArgs)), nil- case "task_type":- taskType, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("task_type value must be a string"))- }- // Validate task type- if _, ok := v1pb.Task_Type_value[taskType]; !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid task_type value: %s", taskType))- }- // Convert v1pb.Task_Type to storepb.Task_Type- v1TaskType := v1pb.Task_Type(v1pb.Task_Type_value[taskType])- storeTaskType := convertToStoreTaskType(v1TaskType)- // Query tasks that have the specified type- positionalArgs = append(positionalArgs, storeTaskType.String())- return fmt.Sprintf("EXISTS (SELECT 1 FROM task WHERE task.pipeline_id = pipeline.id AND task.type = $%d)", len(positionalArgs)), nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported variable %q", variable))- }- case celoperators.In:- variable, value := getVariableAndValueFromExpr(expr)- switch variable {- case "task_type":- rawList, ok := value.([]any)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid list value %q for %v", value, variable))- }- if len(rawList) == 0 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("empty list value for filter %v", variable))- }- var placeholders []string- for _, raw := range rawList {- taskType, ok := raw.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("task_type value must be a string"))- }- // Validate task type- if _, ok := v1pb.Task_Type_value[taskType]; !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid task_type value: %s", taskType))- }- // Convert v1pb.Task_Type to storepb.Task_Type- v1TaskType := v1pb.Task_Type(v1pb.Task_Type_value[taskType])- storeTaskType := convertToStoreTaskType(v1TaskType)- positionalArgs = append(positionalArgs, storeTaskType.String())- placeholders = append(placeholders, fmt.Sprintf("$%d", len(positionalArgs)))- }- // Query tasks that have any of the specified types- return fmt.Sprintf("EXISTS (SELECT 1 FROM task WHERE task.pipeline_id = pipeline.id AND task.type IN (%s))", strings.Join(placeholders, ", ")), nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported variable %q", variable))- }- case celoperators.GreaterEquals, celoperators.LessEquals:- variable, rawValue := getVariableAndValueFromExpr(expr)- value, ok := rawValue.(string)- if !ok {- return "", errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue)- }- if variable != "update_time" {- return "", errors.Errorf(`">=" and "<=" are only supported for "update_time"`)- }- t, err := time.Parse(time.RFC3339, value)- if err != nil {- return "", errors.Errorf("failed to parse time %v, error: %v", value, err)- }- positionalArgs = append(positionalArgs, t)- if functionName == celoperators.GreaterEquals {- return fmt.Sprintf("updated_at >= $%d", len(positionalArgs)), nil- }- return fmt.Sprintf("updated_at <= $%d", len(positionalArgs)), nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported function %v", functionName))- }- default:- return "", errors.Errorf("unexpected expr kind %v", expr.Kind())- }- }-- where, err := getFilter(ast.NativeRep().Expr())+ filterQ, err := s.store.GetListRolloutFilter(ctx, filter)if err != nil {- return err- }-- if where != "" {- pipelineFind.Filter = &store.ListResourceFilter{- Where: "(" + where + ")",- Args: positionalArgs,- }+ return connect.NewError(connect.CodeInvalidArgument, err)}+ pipelineFind.FilterQ = filterQreturn nil}@@ -1184,18 +1069,3 @@ func (s *RolloutService) canUserCancelEnvironmentTaskRun(ctx context.Context, usfunc (s *RolloutService) canUserSkipEnvironmentTasks(ctx context.Context, user *store.UserMessage, project *store.ProjectMessage, issue *store.IssueMessage, environment string, creatorUID int) (bool, error) {return s.canUserRunEnvironmentTasks(ctx, user, project, issue, environment, creatorUID)}--func (s *RolloutService) getUserByIdentifier(ctx context.Context, identifier string) (*store.UserMessage, error) {- email := strings.TrimPrefix(identifier, "users/")- if email == "" {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("invalid empty creator identifier"))- }- user, err := s.store.GetUserByEmail(ctx, email)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf(`failed to find user "%s" with error: %v`, email, err))- }- if user == nil {- return nil, errors.Errorf("cannot found user %s", email)- }- return user, nil-}diff --git a/backend/api/v1/rollout_service_converter.go b/backend/api/v1/rollout_service_converter.goindex 1e8ab6a888..8a21235f67 100644--- a/backend/api/v1/rollout_service_converter.go+++ b/backend/api/v1/rollout_service_converter.go@@ -553,24 +553,6 @@ func convertToTaskType(task *store.TaskMessage) v1pb.Task_Type {}}-func convertToStoreTaskType(taskType v1pb.Task_Type) storepb.Task_Type {- //exhaustive:enforce- switch taskType {- case v1pb.Task_DATABASE_CREATE:- return storepb.Task_DATABASE_CREATE- case v1pb.Task_DATABASE_MIGRATE:- return storepb.Task_DATABASE_MIGRATE- case v1pb.Task_DATABASE_SDL:- return storepb.Task_DATABASE_SDL- case v1pb.Task_DATABASE_EXPORT:- return storepb.Task_DATABASE_EXPORT- case v1pb.Task_TYPE_UNSPECIFIED, v1pb.Task_GENERAL:- return storepb.Task_TASK_TYPE_UNSPECIFIED- default:- return storepb.Task_TASK_TYPE_UNSPECIFIED- }-}-// getMigrateTypeFromMigrationType converts v1pb.MigrationType to storepb.MigrationTypefunc getMigrateTypeFromMigrationType(migrationType v1pb.MigrationType) storepb.MigrationType {switch migrationType {diff --git a/backend/store/pipeline.go b/backend/store/pipeline.goindex 91b3cfe8e4..a86b9810b1 100644--- a/backend/store/pipeline.go+++ b/backend/store/pipeline.go@@ -5,10 +5,15 @@ import ("database/sql""time"+ "github.com/google/cel-go/cel"+ celast "github.com/google/cel-go/common/ast"+ celoperators "github.com/google/cel-go/common/operators""github.com/pkg/errors""github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/qb"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")// PipelineMessage is the message for pipelines.@@ -30,10 +35,9 @@ type PipelineFind struct {ID *intProjectID *string- Limit *int- Offset *int-- Filter *ListResourceFilter+ Limit *int+ Offset *int+ FilterQ *qb.Query}// CreatePipelineAIO creates a pipeline with tasks all in one.@@ -241,8 +245,8 @@ func (s *Store) ListPipelineV2(ctx context.Context, find *PipelineFind) ([]*PipeWHERE TRUE`)- if filter := find.Filter; filter != nil {- q.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)+ if filterQ := find.FilterQ; filterQ != nil {+ q.And("?", filterQ)}if v := find.ID; v != nil {q.And("pipeline.id = ?", *v)@@ -304,3 +308,135 @@ func (s *Store) ListPipelineV2(ctx context.Context, find *PipelineFind) ([]*Pipe}return pipelines, nil}++// GetListRolloutFilter parses a CEL filter expression into a query builder query for listing rollouts.+func (s *Store) GetListRolloutFilter(ctx context.Context, filter string) (*qb.Query, error) {+ if filter == "" {+ return nil, nil+ }++ e, err := cel.NewEnv()+ if err != nil {+ return nil, errors.Errorf("failed to create cel env")+ }+ ast, iss := e.Parse(filter)+ if iss != nil {+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())+ }++ var getFilter func(expr celast.Expr) (*qb.Query, error)++ getFilter = func(expr celast.Expr) (*qb.Query, error) {+ q := qb.Q()+ switch expr.Kind() {+ case celast.CallKind:+ functionName := expr.AsCall().FunctionName()+ switch functionName {+ case celoperators.LogicalAnd:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.And("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.Equals:+ variable, value := getVariableAndValueFromExpr(expr)+ switch variable {+ case "creator":+ user, err := s.getUserByIdentifier(ctx, value.(string))+ if err != nil {+ return nil, errors.Errorf("failed to get user %v with error %v", value, err.Error())+ }+ return qb.Q().Space("pipeline.creator_id = ?", user.ID), nil+ case "task_type":+ taskType, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("task_type value must be a string")+ }+ if _, ok := v1pb.Task_Type_value[taskType]; !ok {+ return nil, errors.Errorf("invalid task_type value: %s", taskType)+ }+ v1TaskType := v1pb.Task_Type(v1pb.Task_Type_value[taskType])+ storeTaskType := convertV1ToStoreTaskType(v1TaskType)+ return qb.Q().Space("EXISTS (SELECT 1 FROM task WHERE task.pipeline_id = pipeline.id AND task.type = ?)", storeTaskType.String()), nil+ default:+ return nil, errors.Errorf("unsupported variable %q", variable)+ }+ case celoperators.In:+ variable, value := getVariableAndValueFromExpr(expr)+ switch variable {+ case "task_type":+ rawList, ok := value.([]any)+ if !ok {+ return nil, errors.Errorf("invalid list value %q for %v", value, variable)+ }+ if len(rawList) == 0 {+ return nil, errors.Errorf("empty list value for filter %v", variable)+ }+ var taskTypes []string+ for _, raw := range rawList {+ taskType, ok := raw.(string)+ if !ok {+ return nil, errors.Errorf("task_type value must be a string")+ }+ if _, ok := v1pb.Task_Type_value[taskType]; !ok {+ return nil, errors.Errorf("invalid task_type value: %s", taskType)+ }+ v1TaskType := v1pb.Task_Type(v1pb.Task_Type_value[taskType])+ storeTaskType := convertV1ToStoreTaskType(v1TaskType)+ taskTypes = append(taskTypes, storeTaskType.String())+ }+ return qb.Q().Space("EXISTS (SELECT 1 FROM task WHERE task.pipeline_id = pipeline.id AND task.type = ANY(?))", taskTypes), nil+ default:+ return nil, errors.Errorf("unsupported variable %q", variable)+ }+ case celoperators.GreaterEquals, celoperators.LessEquals:+ variable, rawValue := getVariableAndValueFromExpr(expr)+ value, ok := rawValue.(string)+ if !ok {+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue)+ }+ if variable != "update_time" {+ return nil, errors.Errorf(`">=" and "<=" are only supported for "update_time"`)+ }+ t, err := time.Parse(time.RFC3339, value)+ if err != nil {+ return nil, errors.Errorf("failed to parse time %v, error: %v", value, err)+ }+ if functionName == celoperators.GreaterEquals {+ return qb.Q().Space("updated_at >= ?", t), nil+ }+ return qb.Q().Space("updated_at <= ?", t), nil+ default:+ return nil, errors.Errorf("unexpected function %v", functionName)+ }+ default:+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())+ }+ }++ q, err := getFilter(ast.NativeRep().Expr())+ if err != nil {+ return nil, err+ }+ return qb.Q().Space("(?)", q), nil+}++func convertV1ToStoreTaskType(taskType v1pb.Task_Type) storepb.Task_Type {+ switch taskType {+ case v1pb.Task_DATABASE_CREATE:+ return storepb.Task_DATABASE_CREATE+ case v1pb.Task_DATABASE_MIGRATE:+ return storepb.Task_DATABASE_MIGRATE+ case v1pb.Task_DATABASE_SDL:+ return storepb.Task_DATABASE_SDL+ case v1pb.Task_DATABASE_EXPORT:+ return storepb.Task_DATABASE_EXPORT+ case v1pb.Task_TYPE_UNSPECIFIED, v1pb.Task_GENERAL:+ return storepb.Task_TASK_TYPE_UNSPECIFIED+ default:+ return storepb.Task_TASK_TYPE_UNSPECIFIED+ }+}