Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(qb): plan filter (#17880)

p0nyf2d2d4b67600
diff --git a/backend/api/v1/plan_service.go b/backend/api/v1/plan_service.go
index 80f20bed69..16ee9aa031 100644
--- a/backend/api/v1/plan_service.go
+++ b/backend/api/v1/plan_service.go
@@ -2,18 +2,15 @@ package v1
import (
"context"
- "fmt"
"log/slog"
"maps"
"slices"
"strings"
- "time"
"connectrpc.com/connect"
"github.com/google/cel-go/cel"
celast "github.com/google/cel-go/common/ast"
celoperators "github.com/google/cel-go/common/operators"
- celoverloads "github.com/google/cel-go/common/overloads"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/common"
@@ -170,9 +167,11 @@ func (s *PlanService) SearchPlans(ctx context.Context, request *connect.Request[
if projectID != "-" {
find.ProjectID = &projectID
}
- if err := s.buildPlanFindWithFilter(ctx, find, req.Filter); err != nil {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to build plan find with filter, error: %v", err))
+ filterQ, err := s.store.GetListPlanFilter(ctx, req.Filter)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
}
+ find.FilterQ = filterQ
plans, err := s.store.ListPlans(ctx, find)
if err != nil {
@@ -1097,161 +1096,6 @@ func (s *PlanService) BatchCancelPlanCheckRuns(ctx context.Context, request *con
return connect.NewResponse(&v1pb.BatchCancelPlanCheckRunsResponse{}), nil
}
-func (s *PlanService) buildPlanFindWithFilter(ctx context.Context, planFind *store.FindPlanMessage, filter string) error {
- if filter == "" {
- return nil
- }
-
- e, err := cel.NewEnv()
- if err != nil {
- return connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))
- }
- ast, iss := e.Parse(filter)
- if iss != nil {
- return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))
- }
-
- var getFilter func(expr celast.Expr) (string, error)
- var positionalArgs []any
-
- getFilter = func(expr celast.Expr) (string, error) {
- switch expr.Kind() {
- case celast.CallKind:
- functionName := expr.AsCall().FunctionName()
- switch functionName {
- case celoperators.LogicalAnd:
- return getSubConditionFromExpr(expr, getFilter, "AND")
- case celoperators.Equals:
- variable, value := getVariableAndValueFromExpr(expr)
- switch variable {
- case "creator":
- user, err := s.getUserByIdentifier(ctx, value.(string))
- if err != nil {
- return "", connect.NewError(connect.CodeInternal, errors.Errorf("failed to get user %v with error %v", value, err.Error()))
- }
- positionalArgs = append(positionalArgs, user.ID)
- return fmt.Sprintf("plan.creator_id = $%d", len(positionalArgs)), nil
- case "has_pipeline":
- hasPipeline, ok := value.(bool)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`"has_pipeline" should be bool`))
- }
- if !hasPipeline {
- return "plan.pipeline_id IS NULL", nil
- }
- return "plan.pipeline_id IS NOT NULL", nil
- case "has_issue":
- hasIssue, ok := value.(bool)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`"has_issue" should be bool`))
- }
- if !hasIssue {
- return "issue.id IS NULL", nil
- }
- return "issue.id IS NOT NULL", nil
- case "title":
- positionalArgs = append(positionalArgs, value)
- return fmt.Sprintf("plan.name = $%d", len(positionalArgs)), nil
- case "spec_type":
- specType, ok := value.(string)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("spec_type value must be a string"))
- }
- switch specType {
- case "create_database_config":
- return "EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'createDatabaseConfig' IS NOT NULL)", nil
- case "change_database_config":
- return "EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'changeDatabaseConfig' IS NOT NULL)", nil
- case "export_data_config":
- return "EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'exportDataConfig' IS NOT NULL)", nil
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid spec_type value: %s, must be one of: create_database_config, change_database_config, export_data_config", specType))
- }
- case "state":
- v1State, ok := v1pb.State_value[value.(string)]
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid state filter %q", value))
- }
- positionalArgs = append(positionalArgs, v1pb.State(v1State) == v1pb.State_DELETED)
- return fmt.Sprintf("plan.deleted = $%d", len(positionalArgs)), nil
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported variable %q", variable))
- }
- case celoperators.GreaterEquals, celoperators.LessEquals:
- variable, rawValue := getVariableAndValueFromExpr(expr)
- value, ok := rawValue.(string)
- if !ok {
- return "", errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue)
- }
- if variable != "create_time" {
- return "", errors.Errorf(`">=" and "<=" are only supported for "create_time"`)
- }
- t, err := time.Parse(time.RFC3339, value)
- if err != nil {
- return "", errors.Errorf("failed to parse time %v, error: %v", value, err)
- }
- positionalArgs = append(positionalArgs, t)
- if functionName == celoperators.GreaterEquals {
- return fmt.Sprintf("plan.created_at >= $%d", len(positionalArgs)), nil
- }
- return fmt.Sprintf("plan.created_at <= $%d", len(positionalArgs)), nil
- case celoverloads.Matches:
- variable := expr.AsCall().Target().AsIdent()
- args := expr.AsCall().Args()
- if len(args) != 1 {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))
- }
- value := args[0].AsLiteral().Value()
- strValue, ok := value.(string)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))
- }
- strValue = strings.ToLower(strValue)
-
- switch variable {
- case "title":
- return "LOWER(plan.name) LIKE '%" + strValue + "%'", nil
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "title" supports %q operator, but found %q`, celoverloads.Matches, variable))
- }
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported function %v", functionName))
- }
- default:
- return "", errors.Errorf("unexpected expr kind %v", expr.Kind())
- }
- }
-
- where, err := getFilter(ast.NativeRep().Expr())
- if err != nil {
- return err
- }
-
- if where != "" {
- planFind.Filter = &store.ListResourceFilter{
- Where: "(" + where + ")",
- Args: positionalArgs,
- }
- }
-
- return nil
-}
-
-func (s *PlanService) getUserByIdentifier(ctx context.Context, identifier string) (*store.UserMessage, error) {
- email := strings.TrimPrefix(identifier, "users/")
- if email == "" {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("invalid empty creator identifier"))
- }
- user, err := s.store.GetUserByEmail(ctx, email)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf(`failed to find user "%s" with error: %v`, email, err))
- }
- if user == nil {
- return nil, errors.Errorf("cannot found user %s", email)
- }
- return user, nil
-}
-
// diffSpecs checks if there are any specs removed, added or updated in the new plan.
// It performs a deep comparison of all spec fields using protocol buffer comparison.
// Returns (removed, added, updated) slices of specs.
diff --git a/backend/store/plan.go b/backend/store/plan.go
index 9845adb78c..947094092e 100644
--- a/backend/store/plan.go
+++ b/backend/store/plan.go
@@ -8,6 +8,10 @@ import (
"strings"
"time"
+ "github.com/google/cel-go/cel"
+ celast "github.com/google/cel-go/common/ast"
+ celoperators "github.com/google/cel-go/common/operators"
+ celoverloads "github.com/google/cel-go/common/overloads"
"github.com/pkg/errors"
"google.golang.org/protobuf/encoding/protojson"
@@ -15,6 +19,7 @@ import (
"github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/common/qb"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)
// PlanMessage is the message for plan.
@@ -42,7 +47,7 @@ type FindPlanMessage struct {
Limit *int
Offset *int
- Filter *ListResourceFilter
+ FilterQ *qb.Query
}
// UpdatePlanMessage is the message to update a plan.
@@ -142,9 +147,8 @@ func (s *Store) ListPlans(ctx context.Context, find *FindPlanMessage) ([]*PlanMe
WHERE TRUE
`)
- if filter := find.Filter; filter != nil {
- // Convert $1, $2, etc. to ? for qb
- q.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)
+ if filterQ := find.FilterQ; filterQ != nil {
+ q.And("?", filterQ)
}
if v := find.UID; v != nil {
q.And("plan.id = ?", *v)
@@ -289,3 +293,164 @@ func (s *Store) UpdatePlan(ctx context.Context, patch *UpdatePlanMessage) error
return nil
}
+
+// GetListPlanFilter parses a CEL filter expression into a query builder query for listing plans.
+func (s *Store) GetListPlanFilter(ctx context.Context, filter string) (*qb.Query, error) {
+ if filter == "" {
+ return nil, nil
+ }
+
+ e, err := cel.NewEnv()
+ if err != nil {
+ return nil, errors.Errorf("failed to create cel env")
+ }
+ ast, iss := e.Parse(filter)
+ if iss != nil {
+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())
+ }
+
+ var getFilter func(expr celast.Expr) (*qb.Query, error)
+
+ getFilter = func(expr celast.Expr) (*qb.Query, error) {
+ q := qb.Q()
+ switch expr.Kind() {
+ case celast.CallKind:
+ functionName := expr.AsCall().FunctionName()
+ switch functionName {
+ case celoperators.LogicalAnd:
+ for _, arg := range expr.AsCall().Args() {
+ qq, err := getFilter(arg)
+ if err != nil {
+ return nil, err
+ }
+ q.And("?", qq)
+ }
+ return qb.Q().Space("(?)", q), nil
+ case celoperators.Equals:
+ variable, value := getVariableAndValueFromExpr(expr)
+ switch variable {
+ case "creator":
+ user, err := s.getUserByIdentifier(ctx, value.(string))
+ if err != nil {
+ return nil, errors.Errorf("failed to get user %v with error %v", value, err.Error())
+ }
+ return qb.Q().Space("plan.creator_id = ?", user.ID), nil
+ case "has_pipeline":
+ hasPipeline, ok := value.(bool)
+ if !ok {
+ return nil, errors.Errorf(`"has_pipeline" should be bool`)
+ }
+ if !hasPipeline {
+ return qb.Q().Space("plan.pipeline_id IS NULL"), nil
+ }
+ return qb.Q().Space("plan.pipeline_id IS NOT NULL"), nil
+ case "has_issue":
+ hasIssue, ok := value.(bool)
+ if !ok {
+ return nil, errors.Errorf(`"has_issue" should be bool`)
+ }
+ if !hasIssue {
+ return qb.Q().Space("issue.id IS NULL"), nil
+ }
+ return qb.Q().Space("issue.id IS NOT NULL"), nil
+ case "title":
+ return qb.Q().Space("plan.name = ?", value), nil
+ case "spec_type":
+ specType, ok := value.(string)
+ if !ok {
+ return nil, errors.Errorf("spec_type value must be a string")
+ }
+ switch specType {
+ case "create_database_config":
+ return qb.Q().Space("EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'createDatabaseConfig' IS NOT NULL)"), nil
+ case "change_database_config":
+ return qb.Q().Space("EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'changeDatabaseConfig' IS NOT NULL)"), nil
+ case "export_data_config":
+ return qb.Q().Space("EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'exportDataConfig' IS NOT NULL)"), nil
+ default:
+ return nil, errors.Errorf("invalid spec_type value: %s, must be one of: create_database_config, change_database_config, export_data_config", specType)
+ }
+ case "state":
+ stateStr, ok := value.(string)
+ if !ok {
+ return nil, errors.Errorf("state value must be string, got %T", value)
+ }
+ v1State, ok := v1pb.State_value[stateStr]
+ if !ok {
+ if v, exists := v1pb.State_value[strings.TrimPrefix(stateStr, "STATE_")]; exists {
+ v1State = v
+ ok = true
+ }
+ }
+ if !ok {
+ return nil, errors.Errorf("invalid state filter %q", value)
+ }
+ return qb.Q().Space("plan.deleted = ?", v1pb.State(v1State) == v1pb.State_DELETED), nil
+ default:
+ return nil, errors.Errorf("unsupported variable %q", variable)
+ }
+ case celoperators.GreaterEquals, celoperators.LessEquals:
+ variable, rawValue := getVariableAndValueFromExpr(expr)
+ value, ok := rawValue.(string)
+ if !ok {
+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue)
+ }
+ if variable != "create_time" {
+ return nil, errors.Errorf(`">=" and "<=" are only supported for "create_time"`)
+ }
+ t, err := time.Parse(time.RFC3339, value)
+ if err != nil {
+ return nil, errors.Errorf("failed to parse time %v, error: %v", value, err)
+ }
+ if functionName == celoperators.GreaterEquals {
+ return qb.Q().Space("plan.created_at >= ?", t), nil
+ }
+ return qb.Q().Space("plan.created_at <= ?", t), nil
+ case celoverloads.Matches:
+ variable := expr.AsCall().Target().AsIdent()
+ args := expr.AsCall().Args()
+ if len(args) != 1 {
+ return nil, errors.Errorf(`invalid args for %q`, variable)
+ }
+ value := args[0].AsLiteral().Value()
+ strValue, ok := value.(string)
+ if !ok {
+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)
+ }
+ strValue = strings.ToLower(strValue)
+
+ switch variable {
+ case "title":
+ return qb.Q().Space("LOWER(plan.name) LIKE ?", "%"+strValue+"%"), nil
+ default:
+ return nil, errors.Errorf(`only "title" supports %q operator, but found %q`, celoverloads.Matches, variable)
+ }
+ default:
+ return nil, errors.Errorf("unsupported function %v", functionName)
+ }
+ default:
+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())
+ }
+ }
+
+ q, err := getFilter(ast.NativeRep().Expr())
+ if err != nil {
+ return nil, err
+ }
+ return qb.Q().Space("(?)", q), nil
+}
+
+func (s *Store) getUserByIdentifier(ctx context.Context, identifier string) (*UserMessage, error) {
+ email := strings.TrimPrefix(identifier, "users/")
+ if email == "" {
+ return nil, errors.New("invalid empty creator identifier")
+ }
+ user, err := s.GetUserByEmail(ctx, email)
+ if err != nil {
+ return nil, errors.Errorf(`failed to find user "%s" with error: %v`, email, err)
+ }
+ if user == nil {
+ return nil, errors.Errorf("cannot found user %s", email)
+ }
+ return user, nil
+}
diff --git a/backend/store/plan_filter_test.go b/backend/store/plan_filter_test.go
new file mode 100644
index 0000000000..11b9a04669
--- /dev/null
+++ b/backend/store/plan_filter_test.go
@@ -0,0 +1,232 @@
+package store
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetListPlanFilter(t *testing.T) {
+ // Create a minimal store instance for testing
+ // Note: Some tests will be limited without a full database connection
+ s := &Store{}
+ ctx := context.Background()
+
+ tests := []struct {
+ name string
+ filter string
+ wantSQL string
+ wantArgs []any
+ wantErr bool
+ errContains string
+ skipTest bool // Skip tests that require database access
+ }{
+ {
+ name: "empty filter",
+ filter: "",
+ wantSQL: "",
+ wantArgs: nil,
+ wantErr: false,
+ },
+ {
+ name: "has_pipeline filter - true",
+ filter: `has_pipeline == true`,
+ wantSQL: "(plan.pipeline_id IS NOT NULL)",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "has_pipeline filter - false",
+ filter: `has_pipeline == false`,
+ wantSQL: "(plan.pipeline_id IS NULL)",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "has_issue filter - true",
+ filter: `has_issue == true`,
+ wantSQL: "(issue.id IS NOT NULL)",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "has_issue filter - false",
+ filter: `has_issue == false`,
+ wantSQL: "(issue.id IS NULL)",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "title filter",
+ filter: `title == "My Plan"`,
+ wantSQL: "(plan.name = $1)",
+ wantArgs: []any{"My Plan"},
+ wantErr: false,
+ },
+ {
+ name: "title matches",
+ filter: `title.matches("test")`,
+ wantSQL: "(LOWER(plan.name) LIKE $1)",
+ wantArgs: []any{"%test%"},
+ wantErr: false,
+ },
+ {
+ name: "spec_type filter - create_database_config",
+ filter: `spec_type == "create_database_config"`,
+ wantSQL: "(EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'createDatabaseConfig' IS NOT NULL))",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "spec_type filter - change_database_config",
+ filter: `spec_type == "change_database_config"`,
+ wantSQL: "(EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'changeDatabaseConfig' IS NOT NULL))",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "spec_type filter - export_data_config",
+ filter: `spec_type == "export_data_config"`,
+ wantSQL: "(EXISTS (SELECT 1 FROM jsonb_array_elements(plan.config->'specs') AS spec WHERE spec->>'exportDataConfig' IS NOT NULL))",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "state filter - ACTIVE",
+ filter: `state == "STATE_ACTIVE"`,
+ wantSQL: "(plan.deleted = $1)",
+ wantArgs: []any{false},
+ wantErr: false,
+ },
+ {
+ name: "state filter - DELETED",
+ filter: `state == "STATE_DELETED"`,
+ wantSQL: "(plan.deleted = $1)",
+ wantArgs: []any{true},
+ wantErr: false,
+ },
+ {
+ name: "create_time greater than or equal",
+ filter: `create_time >= "2024-01-01T00:00:00Z"`,
+ wantSQL: "(plan.created_at >= $1)",
+ wantArgs: []any{func() time.Time {
+ ts, _ := time.Parse(time.RFC3339, "2024-01-01T00:00:00Z")
+ return ts
+ }()},
+ wantErr: false,
+ },
+ {
+ name: "create_time less than or equal",
+ filter: `create_time <= "2024-12-31T23:59:59Z"`,
+ wantSQL: "(plan.created_at <= $1)",
+ wantArgs: []any{func() time.Time {
+ ts, _ := time.Parse(time.RFC3339, "2024-12-31T23:59:59Z")
+ return ts
+ }()},
+ wantErr: false,
+ },
+ {
+ name: "AND condition with has_pipeline and has_issue",
+ filter: `has_pipeline == true && has_issue == true`,
+ wantSQL: "((plan.pipeline_id IS NOT NULL AND issue.id IS NOT NULL))",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "complex AND condition",
+ filter: `title == "Test Plan" && state == "STATE_ACTIVE" && has_pipeline == true`,
+ wantSQL: "(((plan.name = $1 AND plan.deleted = $2) AND plan.pipeline_id IS NOT NULL))",
+ wantArgs: []any{"Test Plan", false},
+ wantErr: false,
+ },
+ {
+ name: "creator filter requires database",
+ filter: `creator == "users/[email protected]"`,
+ skipTest: true,
+ wantErr: false, // Would work with database
+ errContains: "",
+ },
+ {
+ name: "invalid filter syntax",
+ filter: `invalid syntax {{`,
+ wantErr: true,
+ errContains: "failed to parse filter",
+ },
+ {
+ name: "unsupported variable",
+ filter: `unsupported == "value"`,
+ wantErr: true,
+ errContains: "unsupported variable",
+ },
+ {
+ name: "invalid state value",
+ filter: `state == "INVALID_STATE"`,
+ wantErr: true,
+ errContains: "invalid state filter",
+ },
+ {
+ name: "invalid spec_type value",
+ filter: `spec_type == "invalid_type"`,
+ wantErr: true,
+ errContains: "invalid spec_type value",
+ },
+ {
+ name: "has_pipeline with non-bool value",
+ filter: `has_pipeline == "true"`,
+ wantErr: true,
+ errContains: `"has_pipeline" should be bool`,
+ },
+ {
+ name: "has_issue with non-bool value",
+ filter: `has_issue == "true"`,
+ wantErr: true,
+ errContains: `"has_issue" should be bool`,
+ },
+ {
+ name: "invalid time format",
+ filter: `create_time >= "invalid-time"`,
+ wantErr: true,
+ errContains: "failed to parse time",
+ },
+ {
+ name: "comparison operator on unsupported field",
+ filter: `title >= "test"`,
+ wantErr: true,
+ errContains: `">=" and "<=" are only supported for "create_time"`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if tt.skipTest {
+ t.Skip("Test requires database connection")
+ }
+
+ q, err := s.GetListPlanFilter(ctx, tt.filter)
+
+ if tt.wantErr {
+ require.Error(t, err)
+ if tt.errContains != "" {
+ require.ErrorContains(t, err, tt.errContains)
+ }
+ return
+ }
+
+ require.NoError(t, err)
+
+ if tt.filter == "" {
+ require.Nil(t, q)
+ return
+ }
+
+ require.NotNil(t, q)
+
+ sql, args, err := q.ToSQL()
+ require.NoError(t, err)
+ require.Equal(t, tt.wantSQL, sql)
+ require.Equal(t, tt.wantArgs, args)
+ })
+ }
+}