Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(qb): project service filter (#17834)

p0nyd0aa4b4ad1b7
diff --git a/backend/api/v1/project_service.go b/backend/api/v1/project_service.go
index 1ed1a4d7f9..ed2b8194aa 100644
--- a/backend/api/v1/project_service.go
+++ b/backend/api/v1/project_service.go
@@ -11,8 +11,6 @@ import (
"connectrpc.com/connect"
"github.com/google/cel-go/cel"
celast "github.com/google/cel-go/common/ast"
- celoperators "github.com/google/cel-go/common/operators"
- celoverloads "github.com/google/cel-go/common/overloads"
"github.com/gosimple/slug"
"github.com/pkg/errors"
"google.golang.org/genproto/googleapis/type/expr"
@@ -83,11 +81,11 @@ func (s *ProjectService) ListProjects(ctx context.Context, req *connect.Request[
Limit: &limitPlusOne,
Offset: &offset.offset,
}
- filter, err := getListProjectFilter(req.Msg.Filter)
+ filterQ, err := store.GetListProjectFilter(req.Msg.Filter)
if err != nil {
- return nil, err
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
}
- find.Filter = filter
+ find.FilterQ = filterQ
projects, err := s.store.ListProjectV2(ctx, find)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, err)
@@ -110,113 +108,6 @@ func (s *ProjectService) ListProjects(ctx context.Context, req *connect.Request[
return connect.NewResponse(response), nil
}
-func getListProjectFilter(filter string) (*store.ListResourceFilter, error) {
- if filter == "" {
- return nil, nil
- }
- e, err := cel.NewEnv()
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))
- }
- ast, iss := e.Parse(filter)
- if iss != nil {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))
- }
-
- var getFilter func(expr celast.Expr) (string, error)
- var positionalArgs []any
-
- parseToSQL := func(variable, value any) (string, error) {
- switch variable {
- case "name":
- positionalArgs = append(positionalArgs, value.(string))
- return fmt.Sprintf("project.name = $%d", len(positionalArgs)), nil
- case "resource_id":
- positionalArgs = append(positionalArgs, value.(string))
- return fmt.Sprintf("project.resource_id = $%d", len(positionalArgs)), nil
- case "exclude_default":
- if excludeDefault, ok := value.(bool); excludeDefault && ok {
- positionalArgs = append(positionalArgs, common.DefaultProjectID)
- return fmt.Sprintf("project.resource_id != $%d", len(positionalArgs)), nil
- }
- return "TRUE", nil
- case "state":
- v1State, ok := v1pb.State_value[value.(string)]
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid state filter %q", value))
- }
- positionalArgs = append(positionalArgs, v1pb.State(v1State) == v1pb.State_DELETED)
- return fmt.Sprintf("project.deleted = $%d", len(positionalArgs)), nil
- default:
- // Check if it's a label filter (e.g., "labels.environment" == "production")
- varStr, ok := variable.(string)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))
- }
- if labelKey, ok := strings.CutPrefix(varStr, "labels."); ok {
- return parseToLabelFilterSQL("project.setting", labelKey, value)
- }
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))
- }
- }
-
- getFilter = func(expr celast.Expr) (string, error) {
- switch expr.Kind() {
- case celast.CallKind:
- functionName := expr.AsCall().FunctionName()
- switch functionName {
- case celoperators.LogicalOr:
- return getSubConditionFromExpr(expr, getFilter, "OR")
- case celoperators.LogicalAnd:
- return getSubConditionFromExpr(expr, getFilter, "AND")
- case celoperators.Equals:
- variable, value := getVariableAndValueFromExpr(expr)
- return parseToSQL(variable, value)
- case celoverloads.Matches:
- variable := expr.AsCall().Target().AsIdent()
- args := expr.AsCall().Args()
- if len(args) != 1 {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))
- }
- value := args[0].AsLiteral().Value()
- strValue, ok := value.(string)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))
- }
-
- switch variable {
- case "name":
- return "LOWER(project.name) LIKE '%" + strings.ToLower(strValue) + "%'", nil
- case "resource_id":
- return "LOWER(project.resource_id) LIKE '%" + strings.ToLower(strValue) + "%'", nil
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))
- }
- case celoperators.In:
- variable, value := getVariableAndValueFromExpr(expr)
- if labelKey, ok := strings.CutPrefix(variable, "labels."); ok {
- return parseToLabelFilterSQL("project.setting", labelKey, value)
- }
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected %v operator for %v", functionName, variable))
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))
- }
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected expr kind %v", expr.Kind()))
- }
- }
-
- where, err := getFilter(ast.NativeRep().Expr())
- if err != nil {
- return nil, err
- }
-
- return &store.ListResourceFilter{
- Args: positionalArgs,
- Where: "(" + where + ")",
- }, nil
-}
-
// SearchProjects searches all projects on which the user has bb.projects.get permission.
func (s *ProjectService) SearchProjects(ctx context.Context, req *connect.Request[v1pb.SearchProjectsRequest]) (*connect.Response[v1pb.SearchProjectsResponse], error) {
user, ok := GetUserFromContext(ctx)
@@ -239,11 +130,11 @@ func (s *ProjectService) SearchProjects(ctx context.Context, req *connect.Reques
Limit: &limitPlusOne,
Offset: &offset.offset,
}
- filter, err := getListProjectFilter(req.Msg.Filter)
+ filterQ, err := store.GetListProjectFilter(req.Msg.Filter)
if err != nil {
- return nil, err
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
}
- find.Filter = filter
+ find.FilterQ = filterQ
projects, err := s.store.ListProjectV2(ctx, find)
if err != nil {
diff --git a/backend/api/v1/project_service_test.go b/backend/api/v1/project_service_test.go
index 172a63ee89..ec8e152903 100644
--- a/backend/api/v1/project_service_test.go
+++ b/backend/api/v1/project_service_test.go
@@ -6,16 +6,13 @@ import (
"testing"
"time"
- "connectrpc.com/connect"
"github.com/google/go-cmp/cmp"
- "github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/genproto/googleapis/type/expr"
"google.golang.org/protobuf/testing/protocmp"
"google.golang.org/protobuf/types/known/durationpb"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/store"
@@ -373,73 +370,6 @@ func TestFindIamPolicyDeltas(t *testing.T) {
}
}
-func TestListProjectFilter(t *testing.T) {
- testCases := []struct {
- input string
- want *store.ListResourceFilter
- error *connect.Error
- }{
- {
- input: `title == "sample project"`,
- error: connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", "title")),
- },
- {
- input: `name == "sample project"`,
- want: &store.ListResourceFilter{
- Where: `(project.name = $1)`,
- Args: []any{"sample project"},
- },
- },
- {
- input: `name.matches("Sample")`,
- want: &store.ListResourceFilter{
- Where: `(LOWER(project.name) LIKE '%sample%')`,
- },
- },
- {
- input: `resource_id == "sample-project"`,
- want: &store.ListResourceFilter{
- Where: `(project.resource_id = $1)`,
- Args: []any{"sample-project"},
- },
- },
- {
- input: `resource_id.matches("sample")`,
- want: &store.ListResourceFilter{
- Where: `(LOWER(project.resource_id) LIKE '%sample%')`,
- },
- },
- {
- input: `exclude_default == true`,
- want: &store.ListResourceFilter{
- Where: `(project.resource_id != $1)`,
- Args: []any{common.DefaultProjectID},
- },
- },
- {
- input: `(name.matches("sample") || resource_id.matches("Sample")) && exclude_default == true`,
- want: &store.ListResourceFilter{
- Where: `(((LOWER(project.name) LIKE '%sample%') OR (LOWER(project.resource_id) LIKE '%sample%')) AND (project.resource_id != $1))`,
- Args: []any{common.DefaultProjectID},
- },
- },
- }
-
- for _, tc := range testCases {
- filter, err := getListProjectFilter(tc.input)
- if tc.error != nil {
- require.Error(t, err)
- connectErr := new(connect.Error)
- require.True(t, errors.As(err, &connectErr))
- require.Equal(t, tc.error.Message(), connectErr.Message())
- require.Equal(t, tc.error.Code(), connectErr.Code())
- } else {
- require.NoError(t, err)
- require.Equal(t, tc.want.Where, filter.Where)
- }
- }
-}
-
func TestValidateLabels(t *testing.T) {
tests := []struct {
name string
diff --git a/backend/store/project.go b/backend/store/project.go
index d3d5419811..bde11c1e52 100644
--- a/backend/store/project.go
+++ b/backend/store/project.go
@@ -4,7 +4,12 @@ import (
"context"
"database/sql"
"fmt"
+ "strings"
+ "github.com/google/cel-go/cel"
+ celast "github.com/google/cel-go/common/ast"
+ celoperators "github.com/google/cel-go/common/operators"
+ celoverloads "github.com/google/cel-go/common/overloads"
"github.com/pkg/errors"
"google.golang.org/genproto/googleapis/type/expr"
"google.golang.org/protobuf/encoding/protojson"
@@ -12,6 +17,7 @@ import (
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/qb"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)
// ProjectMessage is the message for project.
@@ -34,7 +40,7 @@ type FindProjectMessage struct {
ShowDeleted bool
Limit *int
Offset *int
- Filter *ListResourceFilter
+ FilterQ *qb.Query
}
// UpdateProjectMessage is the message for updating a project.
@@ -279,8 +285,8 @@ func updateProjectImplV2(ctx context.Context, txn *sql.Tx, patch *UpdateProjectM
func (s *Store) listProjectImplV2(ctx context.Context, txn *sql.Tx, find *FindProjectMessage) ([]*ProjectMessage, error) {
q := qb.Q().Space("SELECT resource_id, name, data_classification_config_id, setting, deleted FROM project WHERE TRUE")
- if filter := find.Filter; filter != nil {
- q.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)
+ if filterQ := find.FilterQ; filterQ != nil {
+ q.And("?", filterQ)
}
if v := find.ResourceID; v != nil {
q.And("resource_id = ?", *v)
@@ -573,3 +579,153 @@ func (s *Store) DeleteProject(ctx context.Context, resourceID string) error {
return nil
}
+
+func GetListProjectFilter(filter string) (*qb.Query, error) {
+ if filter == "" {
+ return nil, nil
+ }
+
+ e, err := cel.NewEnv()
+ if err != nil {
+ return nil, errors.Errorf("failed to create cel env")
+ }
+ ast, iss := e.Parse(filter)
+ if iss != nil {
+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())
+ }
+
+ var getFilter func(expr celast.Expr) (*qb.Query, error)
+
+ parseToLabelFilterSQL := func(resource, key string, value any) (*qb.Query, error) {
+ switch v := value.(type) {
+ case string:
+ return qb.Q().Space(fmt.Sprintf("%s->'labels'->>'%s' = ?", resource, key), v), nil
+ case []any:
+ if len(v) == 0 {
+ return nil, errors.Errorf("empty label filter")
+ }
+
+ labelValueList := make([]any, len(v))
+ for i, raw := range v {
+ str, ok := raw.(string)
+ if !ok {
+ return nil, errors.Errorf("label value must be string, got %T", raw)
+ }
+ labelValueList[i] = str
+ }
+ placeholders := strings.Repeat("?,", len(labelValueList))
+ placeholders = placeholders[:len(placeholders)-1]
+ return qb.Q().Space(fmt.Sprintf("%s->'labels'->>'%s' IN (%s)", resource, key, placeholders), labelValueList...), nil
+ default:
+ return nil, errors.Errorf("empty value %v for label filter", value)
+ }
+ }
+
+ parseToSQL := func(variable, value any) (*qb.Query, error) {
+ switch variable {
+ case "name":
+ return qb.Q().Space("project.name = ?", value.(string)), nil
+ case "resource_id":
+ return qb.Q().Space("project.resource_id = ?", value.(string)), nil
+ case "exclude_default":
+ if excludeDefault, ok := value.(bool); excludeDefault && ok {
+ return qb.Q().Space("project.resource_id != ?", common.DefaultProjectID), nil
+ }
+ return qb.Q().Space("TRUE"), nil
+ case "state":
+ stateStr, ok := value.(string)
+ if !ok {
+ return nil, errors.Errorf("state value must be string, got %T", value)
+ }
+ // Try with STATE_ prefix first (e.g., "STATE_ACTIVE", "STATE_DELETED")
+ v1State, ok := v1pb.State_value[stateStr]
+ if !ok {
+ // If not found, try without STATE_ prefix (e.g., "ACTIVE", "DELETED")
+ if v, exists := v1pb.State_value[strings.TrimPrefix(stateStr, "STATE_")]; exists {
+ v1State = v
+ ok = true
+ }
+ }
+ if !ok {
+ return nil, errors.Errorf("invalid state filter %q", value)
+ }
+ return qb.Q().Space("project.deleted = ?", v1pb.State(v1State) == v1pb.State_DELETED), nil
+ default:
+ varStr, ok := variable.(string)
+ if !ok {
+ return nil, errors.Errorf("unsupport variable %q", variable)
+ }
+ if labelKey, ok := strings.CutPrefix(varStr, "labels."); ok {
+ return parseToLabelFilterSQL("project.setting", labelKey, value)
+ }
+ return nil, errors.Errorf("unsupport variable %q", variable)
+ }
+ }
+
+ getFilter = func(expr celast.Expr) (*qb.Query, error) {
+ q := qb.Q()
+ switch expr.Kind() {
+ case celast.CallKind:
+ functionName := expr.AsCall().FunctionName()
+ switch functionName {
+ case celoperators.LogicalOr:
+ for _, arg := range expr.AsCall().Args() {
+ qq, err := getFilter(arg)
+ if err != nil {
+ return nil, err
+ }
+ q.Or("?", qq)
+ }
+ return qb.Q().Space("(?)", q), nil
+ case celoperators.LogicalAnd:
+ for _, arg := range expr.AsCall().Args() {
+ qq, err := getFilter(arg)
+ if err != nil {
+ return nil, err
+ }
+ q.And("?", qq)
+ }
+ return qb.Q().Space("(?)", q), nil
+ case celoperators.Equals:
+ variable, value := getVariableAndValueFromExpr(expr)
+ return parseToSQL(variable, value)
+ case celoverloads.Matches:
+ variable := expr.AsCall().Target().AsIdent()
+ args := expr.AsCall().Args()
+ if len(args) != 1 {
+ return nil, errors.Errorf(`invalid args for %q`, variable)
+ }
+ value := args[0].AsLiteral().Value()
+ strValue, ok := value.(string)
+ if !ok {
+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)
+ }
+
+ switch variable {
+ case "name":
+ return qb.Q().Space("LOWER(project.name) LIKE ?", "%"+strings.ToLower(strValue)+"%"), nil
+ case "resource_id":
+ return qb.Q().Space("LOWER(project.resource_id) LIKE ?", "%"+strings.ToLower(strValue)+"%"), nil
+ default:
+ return nil, errors.Errorf("unsupport variable %q", variable)
+ }
+ case celoperators.In:
+ variable, value := getVariableAndValueFromExpr(expr)
+ if labelKey, ok := strings.CutPrefix(variable, "labels."); ok {
+ return parseToLabelFilterSQL("project.setting", labelKey, value)
+ }
+ return nil, errors.Errorf("unexpected %v operator for %v", functionName, variable)
+ default:
+ return nil, errors.Errorf("unexpected function %v", functionName)
+ }
+ default:
+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())
+ }
+ }
+
+ q, err := getFilter(ast.NativeRep().Expr())
+ if err != nil {
+ return nil, err
+ }
+ return qb.Q().Space("(?)", q), nil
+}
diff --git a/backend/store/project_filter_test.go b/backend/store/project_filter_test.go
new file mode 100644
index 0000000000..313288a2ec
--- /dev/null
+++ b/backend/store/project_filter_test.go
@@ -0,0 +1,163 @@
+package store
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetListProjectFilter(t *testing.T) {
+ tests := []struct {
+ name string
+ filter string
+ wantSQL string
+ wantArgs []any
+ wantErr bool
+ errContains string
+ }{
+ {
+ name: "empty filter",
+ filter: "",
+ wantSQL: "",
+ wantArgs: nil,
+ wantErr: false,
+ },
+ {
+ name: "name filter",
+ filter: `name == "test-project"`,
+ wantSQL: "(project.name = $1)",
+ wantArgs: []any{"test-project"},
+ wantErr: false,
+ },
+ {
+ name: "resource_id filter",
+ filter: `resource_id == "test-project-id"`,
+ wantSQL: "(project.resource_id = $1)",
+ wantArgs: []any{"test-project-id"},
+ wantErr: false,
+ },
+ {
+ name: "exclude_default filter - true",
+ filter: `exclude_default == true`,
+ wantSQL: "(project.resource_id != $1)",
+ wantArgs: []any{"default"},
+ wantErr: false,
+ },
+ {
+ name: "exclude_default filter - false",
+ filter: `exclude_default == false`,
+ wantSQL: "(TRUE)",
+ wantArgs: []any{},
+ wantErr: false,
+ },
+ {
+ name: "state filter - ACTIVE",
+ filter: `state == "STATE_ACTIVE"`,
+ wantSQL: "(project.deleted = $1)",
+ wantArgs: []any{false},
+ wantErr: false,
+ },
+ {
+ name: "state filter - DELETED",
+ filter: `state == "STATE_DELETED"`,
+ wantSQL: "(project.deleted = $1)",
+ wantArgs: []any{true},
+ wantErr: false,
+ },
+ {
+ name: "name matches",
+ filter: `name.matches("test")`,
+ wantSQL: "(LOWER(project.name) LIKE $1)",
+ wantArgs: []any{"%test%"},
+ wantErr: false,
+ },
+ {
+ name: "resource_id matches",
+ filter: `resource_id.matches("prod")`,
+ wantSQL: "(LOWER(project.resource_id) LIKE $1)",
+ wantArgs: []any{"%prod%"},
+ wantErr: false,
+ },
+ {
+ name: "label filter - string",
+ filter: `labels.environment == "production"`,
+ wantSQL: "(project.setting->'labels'->>'environment' = $1)",
+ wantArgs: []any{"production"},
+ wantErr: false,
+ },
+ {
+ name: "label filter - in operator",
+ filter: `labels.environment in ["production", "staging"]`,
+ wantSQL: "(project.setting->'labels'->>'environment' IN ($1,$2))",
+ wantArgs: []any{"production", "staging"},
+ wantErr: false,
+ },
+ {
+ name: "AND condition with name and resource_id",
+ filter: `name == "test-project" && resource_id == "test-id"`,
+ wantSQL: "((project.name = $1 AND project.resource_id = $2))",
+ wantArgs: []any{"test-project", "test-id"},
+ wantErr: false,
+ },
+ {
+ name: "OR condition with name",
+ filter: `name == "project1" || name == "project2"`,
+ wantSQL: "((project.name = $1 OR project.name = $2))",
+ wantArgs: []any{"project1", "project2"},
+ wantErr: false,
+ },
+ {
+ name: "complex nested AND/OR",
+ filter: `(name == "p1" || name == "p2") && exclude_default == true`,
+ wantSQL: "(((project.name = $1 OR project.name = $2) AND project.resource_id != $3))",
+ wantArgs: []any{"p1", "p2", "default"},
+ wantErr: false,
+ },
+ {
+ name: "invalid filter syntax",
+ filter: `invalid syntax {{`,
+ wantErr: true,
+ errContains: "failed to parse filter",
+ },
+ {
+ name: "unsupported variable",
+ filter: `unsupported == "value"`,
+ wantErr: true,
+ errContains: "unsupport variable",
+ },
+ {
+ name: "invalid state value",
+ filter: `state == "INVALID_STATE"`,
+ wantErr: true,
+ errContains: "invalid state filter",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ q, err := GetListProjectFilter(tt.filter)
+
+ if tt.wantErr {
+ require.Error(t, err)
+ if tt.errContains != "" {
+ require.ErrorContains(t, err, tt.errContains)
+ }
+ return
+ }
+
+ require.NoError(t, err)
+
+ if tt.filter == "" {
+ require.Nil(t, q)
+ return
+ }
+
+ require.NotNil(t, q)
+
+ sql, args, err := q.ToSQL()
+ require.NoError(t, err)
+ require.Equal(t, tt.wantSQL, sql)
+ require.Equal(t, tt.wantArgs, args)
+ })
+ }
+}