Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(qb): pipeline (#17883)

p0ny9a9656627f95
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.go
index 0473a01d37..4184794c3c 100644
--- a/backend/api/v1/rollout_service.go
+++ b/backend/api/v1/rollout_service.go
@@ -3,16 +3,12 @@ package v1
import (
"context"
"database/sql"
- "fmt"
"log/slog"
"slices"
"strings"
"time"
"connectrpc.com/connect"
- "github.com/google/cel-go/cel"
- celast "github.com/google/cel-go/common/ast"
- celoperators "github.com/google/cel-go/common/operators"
"github.com/jackc/pgtype"
"github.com/pkg/errors"
"google.golang.org/protobuf/types/known/timestamppb"
@@ -206,123 +202,12 @@ func (s *RolloutService) buildRolloutFindWithFilter(ctx context.Context, pipelin
return nil
}
- e, err := cel.NewEnv()
- if err != nil {
- return connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))
- }
- ast, iss := e.Parse(filter)
- if iss != nil {
- return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))
- }
-
- var getFilter func(expr celast.Expr) (string, error)
- var positionalArgs []any
-
- getFilter = func(expr celast.Expr) (string, error) {
- switch expr.Kind() {
- case celast.CallKind:
- functionName := expr.AsCall().FunctionName()
- switch functionName {
- case celoperators.LogicalAnd:
- return getSubConditionFromExpr(expr, getFilter, "AND")
- case celoperators.Equals:
- variable, value := getVariableAndValueFromExpr(expr)
- switch variable {
- case "creator":
- user, err := s.getUserByIdentifier(ctx, value.(string))
- if err != nil {
- return "", connect.NewError(connect.CodeInternal, errors.Errorf("failed to get user %v with error %v", value, err.Error()))
- }
- positionalArgs = append(positionalArgs, user.ID)
- return fmt.Sprintf("pipeline.creator_id = $%d", len(positionalArgs)), nil
- case "task_type":
- taskType, ok := value.(string)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("task_type value must be a string"))
- }
- // Validate task type
- if _, ok := v1pb.Task_Type_value[taskType]; !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid task_type value: %s", taskType))
- }
- // Convert v1pb.Task_Type to storepb.Task_Type
- v1TaskType := v1pb.Task_Type(v1pb.Task_Type_value[taskType])
- storeTaskType := convertToStoreTaskType(v1TaskType)
- // Query tasks that have the specified type
- positionalArgs = append(positionalArgs, storeTaskType.String())
- return fmt.Sprintf("EXISTS (SELECT 1 FROM task WHERE task.pipeline_id = pipeline.id AND task.type = $%d)", len(positionalArgs)), nil
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported variable %q", variable))
- }
- case celoperators.In:
- variable, value := getVariableAndValueFromExpr(expr)
- switch variable {
- case "task_type":
- rawList, ok := value.([]any)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid list value %q for %v", value, variable))
- }
- if len(rawList) == 0 {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("empty list value for filter %v", variable))
- }
- var placeholders []string
- for _, raw := range rawList {
- taskType, ok := raw.(string)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("task_type value must be a string"))
- }
- // Validate task type
- if _, ok := v1pb.Task_Type_value[taskType]; !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid task_type value: %s", taskType))
- }
- // Convert v1pb.Task_Type to storepb.Task_Type
- v1TaskType := v1pb.Task_Type(v1pb.Task_Type_value[taskType])
- storeTaskType := convertToStoreTaskType(v1TaskType)
- positionalArgs = append(positionalArgs, storeTaskType.String())
- placeholders = append(placeholders, fmt.Sprintf("$%d", len(positionalArgs)))
- }
- // Query tasks that have any of the specified types
- return fmt.Sprintf("EXISTS (SELECT 1 FROM task WHERE task.pipeline_id = pipeline.id AND task.type IN (%s))", strings.Join(placeholders, ", ")), nil
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported variable %q", variable))
- }
- case celoperators.GreaterEquals, celoperators.LessEquals:
- variable, rawValue := getVariableAndValueFromExpr(expr)
- value, ok := rawValue.(string)
- if !ok {
- return "", errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue)
- }
- if variable != "update_time" {
- return "", errors.Errorf(`">=" and "<=" are only supported for "update_time"`)
- }
- t, err := time.Parse(time.RFC3339, value)
- if err != nil {
- return "", errors.Errorf("failed to parse time %v, error: %v", value, err)
- }
- positionalArgs = append(positionalArgs, t)
- if functionName == celoperators.GreaterEquals {
- return fmt.Sprintf("updated_at >= $%d", len(positionalArgs)), nil
- }
- return fmt.Sprintf("updated_at <= $%d", len(positionalArgs)), nil
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported function %v", functionName))
- }
- default:
- return "", errors.Errorf("unexpected expr kind %v", expr.Kind())
- }
- }
-
- where, err := getFilter(ast.NativeRep().Expr())
+ filterQ, err := s.store.GetListRolloutFilter(ctx, filter)
if err != nil {
- return err
- }
-
- if where != "" {
- pipelineFind.Filter = &store.ListResourceFilter{
- Where: "(" + where + ")",
- Args: positionalArgs,
- }
+ return connect.NewError(connect.CodeInvalidArgument, err)
}
+ pipelineFind.FilterQ = filterQ
return nil
}
@@ -1184,18 +1069,3 @@ func (s *RolloutService) canUserCancelEnvironmentTaskRun(ctx context.Context, us
func (s *RolloutService) canUserSkipEnvironmentTasks(ctx context.Context, user *store.UserMessage, project *store.ProjectMessage, issue *store.IssueMessage, environment string, creatorUID int) (bool, error) {
return s.canUserRunEnvironmentTasks(ctx, user, project, issue, environment, creatorUID)
}
-
-func (s *RolloutService) getUserByIdentifier(ctx context.Context, identifier string) (*store.UserMessage, error) {
- email := strings.TrimPrefix(identifier, "users/")
- if email == "" {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("invalid empty creator identifier"))
- }
- user, err := s.store.GetUserByEmail(ctx, email)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf(`failed to find user "%s" with error: %v`, email, err))
- }
- if user == nil {
- return nil, errors.Errorf("cannot found user %s", email)
- }
- return user, nil
-}
diff --git a/backend/api/v1/rollout_service_converter.go b/backend/api/v1/rollout_service_converter.go
index 1e8ab6a888..8a21235f67 100644
--- a/backend/api/v1/rollout_service_converter.go
+++ b/backend/api/v1/rollout_service_converter.go
@@ -553,24 +553,6 @@ func convertToTaskType(task *store.TaskMessage) v1pb.Task_Type {
}
}
-func convertToStoreTaskType(taskType v1pb.Task_Type) storepb.Task_Type {
- //exhaustive:enforce
- switch taskType {
- case v1pb.Task_DATABASE_CREATE:
- return storepb.Task_DATABASE_CREATE
- case v1pb.Task_DATABASE_MIGRATE:
- return storepb.Task_DATABASE_MIGRATE
- case v1pb.Task_DATABASE_SDL:
- return storepb.Task_DATABASE_SDL
- case v1pb.Task_DATABASE_EXPORT:
- return storepb.Task_DATABASE_EXPORT
- case v1pb.Task_TYPE_UNSPECIFIED, v1pb.Task_GENERAL:
- return storepb.Task_TASK_TYPE_UNSPECIFIED
- default:
- return storepb.Task_TASK_TYPE_UNSPECIFIED
- }
-}
-
// getMigrateTypeFromMigrationType converts v1pb.MigrationType to storepb.MigrationType
func getMigrateTypeFromMigrationType(migrationType v1pb.MigrationType) storepb.MigrationType {
switch migrationType {
diff --git a/backend/store/pipeline.go b/backend/store/pipeline.go
index 91b3cfe8e4..a86b9810b1 100644
--- a/backend/store/pipeline.go
+++ b/backend/store/pipeline.go
@@ -5,10 +5,15 @@ import (
"database/sql"
"time"
+ "github.com/google/cel-go/cel"
+ celast "github.com/google/cel-go/common/ast"
+ celoperators "github.com/google/cel-go/common/operators"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/qb"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)
// PipelineMessage is the message for pipelines.
@@ -30,10 +35,9 @@ type PipelineFind struct {
ID *int
ProjectID *string
- Limit *int
- Offset *int
-
- Filter *ListResourceFilter
+ Limit *int
+ Offset *int
+ FilterQ *qb.Query
}
// CreatePipelineAIO creates a pipeline with tasks all in one.
@@ -241,8 +245,8 @@ func (s *Store) ListPipelineV2(ctx context.Context, find *PipelineFind) ([]*Pipe
WHERE TRUE
`)
- if filter := find.Filter; filter != nil {
- q.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)
+ if filterQ := find.FilterQ; filterQ != nil {
+ q.And("?", filterQ)
}
if v := find.ID; v != nil {
q.And("pipeline.id = ?", *v)
@@ -304,3 +308,135 @@ func (s *Store) ListPipelineV2(ctx context.Context, find *PipelineFind) ([]*Pipe
}
return pipelines, nil
}
+
+// GetListRolloutFilter parses a CEL filter expression into a query builder query for listing rollouts.
+func (s *Store) GetListRolloutFilter(ctx context.Context, filter string) (*qb.Query, error) {
+ if filter == "" {
+ return nil, nil
+ }
+
+ e, err := cel.NewEnv()
+ if err != nil {
+ return nil, errors.Errorf("failed to create cel env")
+ }
+ ast, iss := e.Parse(filter)
+ if iss != nil {
+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())
+ }
+
+ var getFilter func(expr celast.Expr) (*qb.Query, error)
+
+ getFilter = func(expr celast.Expr) (*qb.Query, error) {
+ q := qb.Q()
+ switch expr.Kind() {
+ case celast.CallKind:
+ functionName := expr.AsCall().FunctionName()
+ switch functionName {
+ case celoperators.LogicalAnd:
+ for _, arg := range expr.AsCall().Args() {
+ qq, err := getFilter(arg)
+ if err != nil {
+ return nil, err
+ }
+ q.And("?", qq)
+ }
+ return qb.Q().Space("(?)", q), nil
+ case celoperators.Equals:
+ variable, value := getVariableAndValueFromExpr(expr)
+ switch variable {
+ case "creator":
+ user, err := s.getUserByIdentifier(ctx, value.(string))
+ if err != nil {
+ return nil, errors.Errorf("failed to get user %v with error %v", value, err.Error())
+ }
+ return qb.Q().Space("pipeline.creator_id = ?", user.ID), nil
+ case "task_type":
+ taskType, ok := value.(string)
+ if !ok {
+ return nil, errors.Errorf("task_type value must be a string")
+ }
+ if _, ok := v1pb.Task_Type_value[taskType]; !ok {
+ return nil, errors.Errorf("invalid task_type value: %s", taskType)
+ }
+ v1TaskType := v1pb.Task_Type(v1pb.Task_Type_value[taskType])
+ storeTaskType := convertV1ToStoreTaskType(v1TaskType)
+ return qb.Q().Space("EXISTS (SELECT 1 FROM task WHERE task.pipeline_id = pipeline.id AND task.type = ?)", storeTaskType.String()), nil
+ default:
+ return nil, errors.Errorf("unsupported variable %q", variable)
+ }
+ case celoperators.In:
+ variable, value := getVariableAndValueFromExpr(expr)
+ switch variable {
+ case "task_type":
+ rawList, ok := value.([]any)
+ if !ok {
+ return nil, errors.Errorf("invalid list value %q for %v", value, variable)
+ }
+ if len(rawList) == 0 {
+ return nil, errors.Errorf("empty list value for filter %v", variable)
+ }
+ var taskTypes []string
+ for _, raw := range rawList {
+ taskType, ok := raw.(string)
+ if !ok {
+ return nil, errors.Errorf("task_type value must be a string")
+ }
+ if _, ok := v1pb.Task_Type_value[taskType]; !ok {
+ return nil, errors.Errorf("invalid task_type value: %s", taskType)
+ }
+ v1TaskType := v1pb.Task_Type(v1pb.Task_Type_value[taskType])
+ storeTaskType := convertV1ToStoreTaskType(v1TaskType)
+ taskTypes = append(taskTypes, storeTaskType.String())
+ }
+ return qb.Q().Space("EXISTS (SELECT 1 FROM task WHERE task.pipeline_id = pipeline.id AND task.type = ANY(?))", taskTypes), nil
+ default:
+ return nil, errors.Errorf("unsupported variable %q", variable)
+ }
+ case celoperators.GreaterEquals, celoperators.LessEquals:
+ variable, rawValue := getVariableAndValueFromExpr(expr)
+ value, ok := rawValue.(string)
+ if !ok {
+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue)
+ }
+ if variable != "update_time" {
+ return nil, errors.Errorf(`">=" and "<=" are only supported for "update_time"`)
+ }
+ t, err := time.Parse(time.RFC3339, value)
+ if err != nil {
+ return nil, errors.Errorf("failed to parse time %v, error: %v", value, err)
+ }
+ if functionName == celoperators.GreaterEquals {
+ return qb.Q().Space("updated_at >= ?", t), nil
+ }
+ return qb.Q().Space("updated_at <= ?", t), nil
+ default:
+ return nil, errors.Errorf("unexpected function %v", functionName)
+ }
+ default:
+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())
+ }
+ }
+
+ q, err := getFilter(ast.NativeRep().Expr())
+ if err != nil {
+ return nil, err
+ }
+ return qb.Q().Space("(?)", q), nil
+}
+
+func convertV1ToStoreTaskType(taskType v1pb.Task_Type) storepb.Task_Type {
+ switch taskType {
+ case v1pb.Task_DATABASE_CREATE:
+ return storepb.Task_DATABASE_CREATE
+ case v1pb.Task_DATABASE_MIGRATE:
+ return storepb.Task_DATABASE_MIGRATE
+ case v1pb.Task_DATABASE_SDL:
+ return storepb.Task_DATABASE_SDL
+ case v1pb.Task_DATABASE_EXPORT:
+ return storepb.Task_DATABASE_EXPORT
+ case v1pb.Task_TYPE_UNSPECIFIED, v1pb.Task_GENERAL:
+ return storepb.Task_TASK_TYPE_UNSPECIFIED
+ default:
+ return storepb.Task_TASK_TYPE_UNSPECIFIED
+ }
+}