Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(qb): instance filter (#17850)
p0ny40d537021b77
diff --git a/backend/api/v1/instance_service.go b/backend/api/v1/instance_service.goindex 4d72e409d2..35cbbffdeb 100644--- a/backend/api/v1/instance_service.go+++ b/backend/api/v1/instance_service.go@@ -2,15 +2,10 @@ package v1import ("context"- "fmt""log/slog""strings""connectrpc.com/connect"- "github.com/google/cel-go/cel"- celast "github.com/google/cel-go/common/ast"- celoperators "github.com/google/cel-go/common/operators"- celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/emptypb"@@ -70,198 +65,6 @@ func (s *InstanceService) GetInstance(ctx context.Context, req *connect.Request[return connect.NewResponse(result), nil}-func parseListInstanceFilter(filter string) (*store.ListResourceFilter, error) {- if filter == "" {- return nil, nil- }- e, err := cel.NewEnv()- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.New("failed to create cel env"))- }- ast, iss := e.Parse(filter)- if iss != nil {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))- }-- var getFilter func(expr celast.Expr) (string, error)- var positionalArgs []any-- parseToSQL := func(variable, value any) (string, error) {- // Handle label filters like "labels.org_group"- if varStr, ok := variable.(string); ok {- if labelKey, ok := strings.CutPrefix(varStr, "labels."); ok {- return parseToLabelFilterSQL("instance.metadata", labelKey, value)- }- }-- switch variable {- case "name":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("instance.metadata->>'title' = $%d", len(positionalArgs)), nil- case "resource_id":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("instance.resource_id = $%d", len(positionalArgs)), nil- case "environment":- environment, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse value %v to string", value))- }- if environment != "" {- environmentID, err := common.GetEnvironmentID(environment)- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid environment filter %q", value))- }- positionalArgs = append(positionalArgs, environmentID)- return fmt.Sprintf("instance.environment = $%d", len(positionalArgs)), nil- }- return "instance.environment IS NULL", nil- case "state":- v1State, ok := v1pb.State_value[value.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid state filter %q", value))- }- positionalArgs = append(positionalArgs, v1pb.State(v1State) == v1pb.State_DELETED)- return fmt.Sprintf("instance.deleted = $%d", len(positionalArgs)), nil- case "engine":- v1Engine, ok := v1pb.Engine_value[value.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid engine filter %q", value))- }- engine := convertEngine(v1pb.Engine(v1Engine))- positionalArgs = append(positionalArgs, engine)- return fmt.Sprintf("instance.metadata->>'engine' = $%d", len(positionalArgs)), nil- case "host":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("ds ->> 'host' = $%d", len(positionalArgs)), nil- case "port":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("ds ->> 'port' = $%d", len(positionalArgs)), nil- case "project":- projectID, err := common.GetProjectID(value.(string))- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid project filter %q", value))- }- positionalArgs = append(positionalArgs, projectID)- return fmt.Sprintf("db.project = $%d", len(positionalArgs)), nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- }-- getSubConditionFromExpr := func(expr celast.Expr, getFilter func(expr celast.Expr) (string, error), join string) (string, error) {- var args []string- for _, arg := range expr.AsCall().Args() {- filter, err := getFilter(arg)- if err != nil {- return "", err- }- args = append(args, fmt.Sprintf("(%s)", filter))- }- return strings.Join(args, fmt.Sprintf(" %s ", join)), nil- }-- parseToEngineSQL := func(expr celast.Expr, relation string) (string, error) {- variable, value := getVariableAndValueFromExpr(expr)- if variable != "engine" {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "engine" support "engine in [xx]"/"!(engine in [xx])" operator`))- }- if value == nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`empty value %v for "%s" operator`, value, relation))- }- list, ok := value.([]any)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`expect list, got %T, hint: filter engine in ["xx"]`, value))- }- if len(list) == 0 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`empty value %v for "%s" operator`, value, relation))- }-- var engineList []string- for _, raw := range list {- engine, ok := raw.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`expect string, got %T for engine %v`, raw, raw))- }- v1Engine, ok := v1pb.Engine_value[engine]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid engine filter %q`, engine))- }- storeEngine := convertEngine(v1pb.Engine(v1Engine))- engineList = append(engineList, fmt.Sprintf("'%s'", storeEngine))- }- return fmt.Sprintf("instance.metadata->>'engine' %s (%s)", relation, strings.Join(engineList, ",")), nil- }-- getFilter = func(expr celast.Expr) (string, error) {- switch expr.Kind() {- case celast.CallKind:- functionName := expr.AsCall().FunctionName()- switch functionName {- case celoperators.LogicalOr:- return getSubConditionFromExpr(expr, getFilter, "OR")- case celoperators.LogicalAnd:- return getSubConditionFromExpr(expr, getFilter, "AND")- case celoperators.Equals:- variable, value := getVariableAndValueFromExpr(expr)- return parseToSQL(variable, value)- case celoverloads.Matches:- variable := expr.AsCall().Target().AsIdent()- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))- }- value := args[0].AsLiteral().Value()- strValue, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))- }- if strValue == "" {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`empty value for %q`, variable))- }-- switch variable {- case "name":- return "LOWER(instance.metadata->>'title') LIKE '%" + strings.ToLower(strValue) + "%'", nil- case "resource_id":- return "LOWER(instance.resource_id) LIKE '%" + strings.ToLower(strValue) + "%'", nil- case "host", "port":- return "ds ->> '" + variable + "' LIKE '%" + strValue + "%'", nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- case celoperators.In:- variable, value := getVariableAndValueFromExpr(expr)- if variable == "engine" {- return parseToEngineSQL(expr, "IN")- } else if labelKey, ok := strings.CutPrefix(variable, "labels."); ok {- return parseToLabelFilterSQL("instance.metadata", labelKey, value)- }- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- case celoperators.LogicalNot:- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.New(`only support !(engine in ["{engine1}", "{engine2}"]) format`))- }- return parseToEngineSQL(args[0], "NOT IN")- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))- }- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected expr kind %v", expr.Kind()))- }- }-- where, err := getFilter(ast.NativeRep().Expr())- if err != nil {- return nil, err- }-- return &store.ListResourceFilter{- Args: positionalArgs,- Where: "(" + where + ")",- }, nil-}-// ListInstances lists all instances.func (s *InstanceService) ListInstances(ctx context.Context, req *connect.Request[v1pb.ListInstancesRequest]) (*connect.Response[v1pb.ListInstancesResponse], error) {offset, err := parseLimitAndOffset(&pageSize{@@ -279,11 +82,11 @@ func (s *InstanceService) ListInstances(ctx context.Context, req *connect.RequesLimit: &limitPlusOne,Offset: &offset.offset,}- filter, err := parseListInstanceFilter(req.Msg.Filter)+ filterQ, err := store.GetListInstanceFilter(req.Msg.Filter)if err != nil {- return nil, err+ return nil, connect.NewError(connect.CodeInvalidArgument, err)}- find.Filter = filter+ find.FilterQ = filterQinstances, err := s.store.ListInstancesV2(ctx, find)if err != nil {return nil, connect.NewError(connect.CodeInternal, err)diff --git a/backend/store/instance.go b/backend/store/instance.goindex d2a0a8277f..f04506a186 100644--- a/backend/store/instance.go+++ b/backend/store/instance.go@@ -3,16 +3,21 @@ package storeimport ("context""database/sql"+ "fmt""strings"+ "github.com/google/cel-go/cel"+ celast "github.com/google/cel-go/common/ast"+ celoperators "github.com/google/cel-go/common/operators"+ celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors"-"google.golang.org/protobuf/encoding/protojson""google.golang.org/protobuf/proto""github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/qb"storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")// InstanceMessage is the message for instance.@@ -41,7 +46,7 @@ type FindInstanceMessage struct {ShowDeleted boolLimit *intOffset *int- Filter *ListResourceFilter+ FilterQ *qb.Query}// GetInstanceV2 gets an instance by the resource_id.@@ -222,12 +227,13 @@ func (s *Store) UpdateInstanceV2(ctx context.Context, patch *UpdateInstanceMessafunc (s *Store) listInstanceImplV2(ctx context.Context, txn *sql.Tx, find *FindInstanceMessage) ([]*InstanceMessage, error) {where := qb.Q().Space("TRUE")from := qb.Q().Space("instance")- if filter := find.Filter; filter != nil {- where.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)- if hasHostPortFilter(filter.Where) {+ if filterQ := find.FilterQ; filterQ != nil {+ where.And("?", filterQ)+ filterSQL, _, _ := filterQ.ToSQL()+ if hasHostPortFilter(filterSQL) {from.Space("CROSS JOIN jsonb_array_elements(instance.metadata -> 'dataSources') AS ds")}- if strings.Contains(filter.Where, "db.project") {+ if strings.Contains(filterSQL, "db.project") {from.Space("LEFT JOIN db ON db.instance = instance.resource_id")}}@@ -683,3 +689,215 @@ func (s *Store) DeleteInstance(ctx context.Context, resourceID string) error {return nil}++func GetListInstanceFilter(filter string) (*qb.Query, error) {+ if filter == "" {+ return nil, nil+ }++ e, err := cel.NewEnv()+ if err != nil {+ return nil, errors.Errorf("failed to create cel env")+ }+ ast, iss := e.Parse(filter)+ if iss != nil {+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())+ }++ var getFilter func(expr celast.Expr) (*qb.Query, error)++ parseToLabelFilterSQL := func(resource, key string, value any) (*qb.Query, error) {+ switch v := value.(type) {+ case string:+ return qb.Q().Space(fmt.Sprintf("%s->'labels'->>'%s' = ?", resource, key), v), nil+ case []any:+ if len(v) == 0 {+ return nil, errors.Errorf("empty label filter")+ }++ labelValueList := make([]any, len(v))+ for i, raw := range v {+ str, ok := raw.(string)+ if !ok {+ return nil, errors.Errorf("label value must be string, got %T", raw)+ }+ labelValueList[i] = str+ }+ placeholders := strings.Repeat("?,", len(labelValueList))+ placeholders = placeholders[:len(placeholders)-1]+ return qb.Q().Space(fmt.Sprintf("%s->'labels'->>'%s' IN (%s)", resource, key, placeholders), labelValueList...), nil+ default:+ return nil, errors.Errorf("empty value %v for label filter", value)+ }+ }++ parseToSQL := func(variable, value any) (*qb.Query, error) {+ // Handle label filters like "labels.org_group"+ if varStr, ok := variable.(string); ok {+ if labelKey, ok := strings.CutPrefix(varStr, "labels."); ok {+ return parseToLabelFilterSQL("instance.metadata", labelKey, value)+ }+ }++ switch variable {+ case "name":+ return qb.Q().Space("instance.metadata->>'title' = ?", value.(string)), nil+ case "resource_id":+ return qb.Q().Space("instance.resource_id = ?", value.(string)), nil+ case "environment":+ environment, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("failed to parse value %v to string", value)+ }+ if environment != "" {+ environmentID, err := common.GetEnvironmentID(environment)+ if err != nil {+ return nil, errors.Errorf("invalid environment filter %q", value)+ }+ return qb.Q().Space("instance.environment = ?", environmentID), nil+ }+ return qb.Q().Space("instance.environment IS NULL"), nil+ case "state":+ v1State, ok := v1pb.State_value[value.(string)]+ if !ok {+ return nil, errors.Errorf("invalid state filter %q", value)+ }+ return qb.Q().Space("instance.deleted = ?", v1pb.State(v1State) == v1pb.State_DELETED), nil+ case "engine":+ v1Engine, ok := v1pb.Engine_value[value.(string)]+ if !ok {+ return nil, errors.Errorf("invalid engine filter %q", value)+ }+ engine := convertEngine(v1pb.Engine(v1Engine))+ return qb.Q().Space("instance.metadata->>'engine' = ?", engine), nil+ case "host":+ return qb.Q().Space("ds ->> 'host' = ?", value.(string)), nil+ case "port":+ return qb.Q().Space("ds ->> 'port' = ?", value.(string)), nil+ case "project":+ projectID, err := common.GetProjectID(value.(string))+ if err != nil {+ return nil, errors.Errorf("invalid project filter %q", value)+ }+ return qb.Q().Space("db.project = ?", projectID), nil+ default:+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ }++ parseToEngineSQL := func(expr celast.Expr, relation string) (*qb.Query, error) {+ variable, value := getVariableAndValueFromExpr(expr)+ if variable != "engine" {+ return nil, errors.Errorf(`only "engine" support "engine in [xx]"/"!(engine in [xx])" operator`)+ }+ if value == nil {+ return nil, errors.Errorf(`empty value %v for "%s" operator`, value, relation)+ }+ list, ok := value.([]any)+ if !ok {+ return nil, errors.Errorf(`expect list, got %T, hint: filter engine in ["xx"]`, value)+ }+ if len(list) == 0 {+ return nil, errors.Errorf(`empty value %v for "%s" operator`, value, relation)+ }++ var engineList []string+ for _, raw := range list {+ engine, ok := raw.(string)+ if !ok {+ return nil, errors.Errorf(`expect string, got %T for engine %v`, raw, raw)+ }+ v1Engine, ok := v1pb.Engine_value[engine]+ if !ok {+ return nil, errors.Errorf(`invalid engine filter %q`, engine)+ }+ storeEngine := convertEngine(v1pb.Engine(v1Engine))+ engineList = append(engineList, fmt.Sprintf("'%s'", storeEngine))+ }+ return qb.Q().Space(fmt.Sprintf("instance.metadata->>'engine' %s (%s)", relation, strings.Join(engineList, ","))), nil+ }++ getFilter = func(expr celast.Expr) (*qb.Query, error) {+ q := qb.Q()+ switch expr.Kind() {+ case celast.CallKind:+ functionName := expr.AsCall().FunctionName()+ switch functionName {+ case celoperators.LogicalOr:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.Or("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.LogicalAnd:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.And("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.Equals:+ variable, value := getVariableAndValueFromExpr(expr)+ return parseToSQL(variable, value)+ case celoverloads.Matches:+ variable := expr.AsCall().Target().AsIdent()+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`invalid args for %q`, variable)+ }+ value := args[0].AsLiteral().Value()+ strValue, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)+ }+ if strValue == "" {+ return nil, errors.Errorf(`empty value for %q`, variable)+ }++ switch variable {+ case "name":+ return qb.Q().Space("LOWER(instance.metadata->>'title') LIKE ?", "%"+strings.ToLower(strValue)+"%"), nil+ case "resource_id":+ return qb.Q().Space("LOWER(instance.resource_id) LIKE ?", "%"+strings.ToLower(strValue)+"%"), nil+ case "host", "port":+ return qb.Q().Space(fmt.Sprintf("ds ->> '%s' LIKE ?", variable), "%"+strValue+"%"), nil+ default:+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ case celoperators.In:+ variable, value := getVariableAndValueFromExpr(expr)+ if variable == "engine" {+ return parseToEngineSQL(expr, "IN")+ } else if labelKey, ok := strings.CutPrefix(variable, "labels."); ok {+ return parseToLabelFilterSQL("instance.metadata", labelKey, value)+ }+ return nil, errors.Errorf("unsupport variable %q", variable)+ case celoperators.LogicalNot:+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`only support !(engine in ["{engine1}", "{engine2}"]) format`)+ }+ return parseToEngineSQL(args[0], "NOT IN")+ default:+ return nil, errors.Errorf("unexpected function %v", functionName)+ }+ default:+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())+ }+ }++ q, err := getFilter(ast.NativeRep().Expr())+ if err != nil {+ return nil, err+ }+ return qb.Q().Space("(?)", q), nil+}++func convertEngine(engine v1pb.Engine) string {+ return storepb.Engine_name[int32(engine)]+}