Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(qb): database filter (#17888)
p0ny2b1dd62a8d06
diff --git a/backend/api/v1/database_service.go b/backend/api/v1/database_service.goindex c6b7eda001..3bc1eb292f 100644--- a/backend/api/v1/database_service.go+++ b/backend/api/v1/database_service.go@@ -142,217 +142,6 @@ func getSubConditionFromExpr(expr celast.Expr, getFilter func(expr celast.Expr)return strings.Join(args, fmt.Sprintf(" %s ", join)), nil}-func parseToLabelFilterSQL(resource, key string, value any) (string, error) {- switch v := value.(type) {- case string:- return fmt.Sprintf("%s->'labels'->>'%s' = '%s'", resource, key, v), nil- case []any:- if len(v) == 0 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("empty label filter"))- }-- labelValueList := []string{}- for _, raw := range v {- labelValueList = append(labelValueList, fmt.Sprintf(`'%s'`, raw.(string)))- }- return fmt.Sprintf("%s->'labels'->>'%s' IN (%s)", resource, key, strings.Join(labelValueList, ",")), nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("empty value %v for label filter", value))- }-}--func parseToEngineSQL(expr celast.Expr, relation string) (string, error) {- variable, value := getVariableAndValueFromExpr(expr)- if variable != "engine" {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "engine" support "engine in [xx]"/"!(engine in [xx])" operator`))- }-- rawEngineList, ok := value.([]any)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid engine value %q", value))- }- if len(rawEngineList) == 0 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("empty engine filter"))- }- engineList := []string{}- for _, rawEngine := range rawEngineList {- v1Engine, ok := v1pb.Engine_value[rawEngine.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid engine filter %q", rawEngine))- }- engine := convertEngine(v1pb.Engine(v1Engine))- engineList = append(engineList, fmt.Sprintf(`'%s'`, engine.String()))- }-- return fmt.Sprintf("instance.metadata->>'engine' %s (%s)", relation, strings.Join(engineList, ",")), nil-}--func getListDatabaseFilter(filter string) (*store.ListResourceFilter, error) {- if filter == "" {- return nil, nil- }-- e, err := cel.NewEnv()- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))- }- ast, iss := e.Parse(filter)- if iss != nil {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))- }-- var getFilter func(expr celast.Expr) (string, error)- var positionalArgs []any-- parseToSQL := func(variable, value any) (string, error) {- switch variable {- case "project":- projectID, err := common.GetProjectID(value.(string))- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid project filter %q", value))- }- positionalArgs = append(positionalArgs, projectID)- return fmt.Sprintf("db.project = $%d", len(positionalArgs)), nil- case "instance":- instanceID, err := common.GetInstanceID(value.(string))- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid instance filter %q", value))- }- positionalArgs = append(positionalArgs, instanceID)- return fmt.Sprintf("db.instance = $%d", len(positionalArgs)), nil- case "environment":- environment, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse value %v to string", value))- }- if environment != "" {- environmentID, err := common.GetEnvironmentID(environment)- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid environment filter %q", value))- }- positionalArgs = append(positionalArgs, environmentID)- return fmt.Sprintf(`COALESCE(db.environment, instance.environment) = $%d`, len(positionalArgs)), nil- }- return "db.environment IS NULL AND instance.environment IS NULL", nil- case "engine":- v1Engine, ok := v1pb.Engine_value[value.(string)]- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid engine filter %q", value))- }- engine := convertEngine(v1pb.Engine(v1Engine))- positionalArgs = append(positionalArgs, engine)- return fmt.Sprintf("instance.metadata->>'engine' = $%d", len(positionalArgs)), nil- case "name":- positionalArgs = append(positionalArgs, value)- return fmt.Sprintf("db.name = $%d", len(positionalArgs)), nil- case "drifted":- drifted, ok := value.(bool)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid drifted filter %q", value))- }- condition := "IS"- if !drifted {- condition = "IS NOT"- }- return fmt.Sprintf("(db.metadata->>'drifted')::boolean %s TRUE", condition), nil- case "exclude_unassigned":- if excludeUnassigned, ok := value.(bool); excludeUnassigned && ok {- positionalArgs = append(positionalArgs, common.DefaultProjectID)- return fmt.Sprintf("db.project != $%d", len(positionalArgs)), nil- }- return "TRUE", nil- case "table":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf(`- EXISTS (- SELECT 1- FROM json_array_elements(ds.metadata->'schemas') AS s,- json_array_elements(s->'tables') AS t- WHERE t->>'name' = $%d- )- `, len(positionalArgs)), nil- default:- varStr, ok := variable.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- if labelKey, ok := strings.CutPrefix(varStr, "labels."); ok {- return parseToLabelFilterSQL("db.metadata", labelKey, value)- }- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- }-- getFilter = func(expr celast.Expr) (string, error) {- switch expr.Kind() {- case celast.CallKind:- functionName := expr.AsCall().FunctionName()- switch functionName {- case celoperators.LogicalOr:- return getSubConditionFromExpr(expr, getFilter, "OR")- case celoperators.LogicalAnd:- return getSubConditionFromExpr(expr, getFilter, "AND")- case celoperators.Equals:- variable, value := getVariableAndValueFromExpr(expr)- return parseToSQL(variable, value)- case celoverloads.Matches:- variable := expr.AsCall().Target().AsIdent()- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))- }- value := args[0].AsLiteral().Value()- strValue, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))- }- strValue = strings.ToLower(strValue)-- switch variable {- case "name":- return "LOWER(db.name) LIKE '%" + strValue + "%'", nil- case "table":- return `EXISTS (- SELECT 1- FROM json_array_elements(ds.metadata->'schemas') AS s,- json_array_elements(s->'tables') AS t- WHERE t->>'name' LIKE '%` + strValue + `%')`, nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "name" or "table" support %q operator, but found %q`, celoverloads.Matches, variable))- }- case celoperators.In:- variable, value := getVariableAndValueFromExpr(expr)- if variable == "engine" {- return parseToEngineSQL(expr, "IN")- } else if labelKey, ok := strings.CutPrefix(variable, "labels."); ok {- return parseToLabelFilterSQL("db.metadata", labelKey, value)- }- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- case celoperators.LogicalNot:- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only support !(engine in ["{engine1}", "{engine2}"]) format`))- }- return parseToEngineSQL(args[0], "NOT IN")- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))- }- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected expr kind %v", expr.Kind()))- }- }-- where, err := getFilter(ast.NativeRep().Expr())- if err != nil {- return nil, err- }-- return &store.ListResourceFilter{- Args: positionalArgs,- Where: "(" + where + ")",- }, nil-}-// ListDatabases lists all databases.func (s *DatabaseService) ListDatabases(ctx context.Context, req *connect.Request[v1pb.ListDatabasesRequest]) (*connect.Response[v1pb.ListDatabasesResponse], error) {user, ok := GetUserFromContext(ctx)@@ -376,11 +165,11 @@ func (s *DatabaseService) ListDatabases(ctx context.Context, req *connect.RequesShowDeleted: req.Msg.ShowDeleted,}- filter, err := getListDatabaseFilter(req.Msg.Filter)+ filterQ, err := store.GetListDatabaseFilter(req.Msg.Filter)if err != nil {- return nil, err+ return nil, connect.NewError(connect.CodeInvalidArgument, err)}- find.Filter = filter+ find.FilterQ = filterQswitch {case strings.HasPrefix(req.Msg.Parent, common.ProjectNamePrefix):diff --git a/backend/api/v1/database_service_test.go b/backend/api/v1/database_service_test.goindex fb6bd14d47..8e853cd463 100644--- a/backend/api/v1/database_service_test.go+++ b/backend/api/v1/database_service_test.go@@ -1,7 +1,6 @@package v1import (- "fmt""testing""connectrpc.com/connect"@@ -15,75 +14,63 @@ import (func TestListDatabaseFilter(t *testing.T) {testCases := []struct {- input string- want *store.ListResourceFilter- error *connect.Error+ input string+ wantSQL string+ wantArgs []any+ error *connect.Error}{{- input: `environment == "environments/test"`,- want: &store.ListResourceFilter{- Where: `(COALESCE(db.environment, instance.environment) = $1)`,- Args: []any{"test"},- },+ input: `environment == "environments/test"`,+ wantSQL: `(COALESCE(db.environment, instance.environment) = $1)`,+ wantArgs: []any{"test"},},{input: `environment == "test"`,error: connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid environment filter %q", "test")),},{- input: `project == "projects/sample"`,- want: &store.ListResourceFilter{- Where: "(db.project = $1)",- Args: []any{"sample"},- },+ input: `project == "projects/sample"`,+ wantSQL: `(db.project = $1)`,+ wantArgs: []any{"sample"},},{- input: `name.matches("Employee")`,- want: &store.ListResourceFilter{- Where: `(LOWER(db.name) LIKE '%employee%')`,- },+ input: `name.matches("Employee")`,+ wantSQL: `(LOWER(db.name) LIKE $1)`,+ wantArgs: []any{"%employee%"},},{- input: `engine in ["MYSQL", "POSTGRES"]`,- want: &store.ListResourceFilter{- Where: fmt.Sprintf(`(instance.metadata->>'engine' IN ('%s','%s'))`, v1pb.Engine_MYSQL.String(), v1pb.Engine_POSTGRES.String()),- Args: []any{},- },+ input: `engine in ["MYSQL", "POSTGRES"]`,+ wantSQL: `(instance.metadata->>'engine' = ANY($1))`,+ wantArgs: []any{[]any{v1pb.Engine_MYSQL.String(), v1pb.Engine_POSTGRES.String()}},},{- input: `labels.region == "asia" && labels.tenant == "bytebase"`,- want: &store.ListResourceFilter{- Where: `((db.metadata->'labels'->>'region' = 'asia') AND (db.metadata->'labels'->>'tenant' = 'bytebase'))`,- Args: []any{},- },+ input: `labels.region == "asia" && labels.tenant == "bytebase"`,+ wantSQL: `((db.metadata->'labels'->>'region' = $1 AND db.metadata->'labels'->>'tenant' = $2))`,+ wantArgs: []any{"asia", "bytebase"},},{- input: `(labels.region == "asia" || labels.tenant == "bytebase") && exclude_unassigned == true`,- want: &store.ListResourceFilter{- Where: `(((db.metadata->'labels'->>'region' = 'asia') OR (db.metadata->'labels'->>'tenant' = 'bytebase')) AND (db.project != $1))`,- Args: []any{common.DefaultProjectID},- },+ input: `(labels.region == "asia" || labels.tenant == "bytebase") && exclude_unassigned == true`,+ wantSQL: `(((db.metadata->'labels'->>'region' = $1 OR db.metadata->'labels'->>'tenant' = $2) AND db.project != $3))`,+ wantArgs: []any{"asia", "bytebase", common.DefaultProjectID},},{- input: `labels.region in ["asia", "europe"] && labels.tenant == "bytebase"`,- want: &store.ListResourceFilter{- Where: `((db.metadata->'labels'->>'region' IN ('asia','europe')) AND (db.metadata->'labels'->>'tenant' = 'bytebase'))`,- Args: []any{},- },+ input: `labels.region in ["asia", "europe"] && labels.tenant == "bytebase"`,+ wantSQL: `((db.metadata->'labels'->>'region' = ANY($1) AND db.metadata->'labels'->>'tenant' = $2))`,+ wantArgs: []any{[]any{"asia", "europe"}, "bytebase"},},}for _, tc := range testCases {- filter, err := getListDatabaseFilter(tc.input)+ filterQ, err := store.GetListDatabaseFilter(tc.input)if tc.error != nil {require.Error(t, err)- connectErr := new(connect.Error)- require.True(t, errors.As(err, &connectErr))- require.Equal(t, tc.error.Message(), connectErr.Message())- require.Equal(t, tc.error.Code(), connectErr.Code())+ require.Equal(t, tc.error.Message(), err.Error())} else {require.NoError(t, err)- require.Equal(t, tc.want.Where, filter.Where)+ sql, args, err := filterQ.ToSQL()+ require.NoError(t, err)+ require.Equal(t, tc.wantSQL, sql)+ require.Equal(t, tc.wantArgs, args)}}}diff --git a/backend/store/database.go b/backend/store/database.goindex 6032d8fd5f..e0253f7c70 100644--- a/backend/store/database.go+++ b/backend/store/database.go@@ -3,8 +3,13 @@ package storeimport ("context""database/sql"+ "fmt""strings"+ "github.com/google/cel-go/cel"+ celast "github.com/google/cel-go/common/ast"+ celoperators "github.com/google/cel-go/common/operators"+ celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson""google.golang.org/protobuf/proto"@@ -65,9 +70,9 @@ type FindDatabaseMessage struct {// IsCaseSensitive is used to ignore case sensitive when finding database.IsCaseSensitive bool- Filter *ListResourceFilter- Limit *int- Offset *int+ FilterQ *qb.Query+ Limit *int+ Offset *int}// GetDatabaseV2 gets a database.@@ -394,9 +399,11 @@ func (*Store) listDatabaseImplV2(ctx context.Context, txn *sql.Tx, find *FindDatfrom := qb.Q().Space("db")where := qb.Q().Space("TRUE")- if filter := find.Filter; filter != nil {- where.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)- if strings.Contains(filter.Where, "ds.metadata->'schemas'") {+ if filterQ := find.FilterQ; filterQ != nil {+ where.And("?", filterQ)+ // Check if the filter requires the db_schema table for table filtering+ sql, _, err := filterQ.ToSQL()+ if err == nil && strings.Contains(sql, "ds.metadata") {from.Space("INNER JOIN db_schema ds ON db.instance = ds.instance AND db.name = ds.db_name")}}@@ -501,3 +508,220 @@ func (*Store) listDatabaseImplV2(ctx context.Context, txn *sql.Tx, find *FindDatreturn databaseMessages, nil}++func GetListDatabaseFilter(filter string) (*qb.Query, error) {+ if filter == "" {+ return nil, nil+ }++ e, err := cel.NewEnv()+ if err != nil {+ return nil, errors.Errorf("failed to create cel env")+ }+ ast, iss := e.Parse(filter)+ if iss != nil {+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())+ }++ var getFilter func(expr celast.Expr) (*qb.Query, error)++ parseToLabelFilterSQL := func(resource, key string, value any) (*qb.Query, error) {+ switch v := value.(type) {+ case string:+ return qb.Q().Space(fmt.Sprintf("%s->'labels'->>'%s' = ?", resource, key), v), nil+ case []any:+ if len(v) == 0 {+ return nil, errors.Errorf("empty label filter")+ }+ labelValueList := []any{}+ for _, raw := range v {+ labelValueList = append(labelValueList, raw.(string))+ }+ return qb.Q().Space(fmt.Sprintf("%s->'labels'->>'%s' = ANY(?)", resource, key), labelValueList), nil+ default:+ return nil, errors.Errorf("empty value %v for label filter", value)+ }+ }++ parseToEngineSQL := func(expr celast.Expr) (*qb.Query, error) {+ variable, value := getVariableAndValueFromExpr(expr)+ if variable != "engine" {+ return nil, errors.Errorf(`only "engine" support "engine in [xx]"/"!(engine in [xx])" operator`)+ }++ rawEngineList, ok := value.([]any)+ if !ok {+ return nil, errors.Errorf("invalid engine value %q", value)+ }+ if len(rawEngineList) == 0 {+ return nil, errors.Errorf("empty engine filter")+ }+ engineList := []any{}+ for _, rawEngine := range rawEngineList {+ engineValue, ok := storepb.Engine_value[rawEngine.(string)]+ if !ok {+ return nil, errors.Errorf("invalid engine filter %q", rawEngine)+ }+ engine := storepb.Engine(engineValue)+ engineList = append(engineList, engine.String())+ }++ return qb.Q().Space("instance.metadata->>'engine' = ANY(?)", engineList), nil+ }++ parseToSQL := func(variable, value any) (*qb.Query, error) {+ switch variable {+ case "project":+ projectID, err := common.GetProjectID(value.(string))+ if err != nil {+ return nil, errors.Errorf("invalid project filter %q", value)+ }+ return qb.Q().Space("db.project = ?", projectID), nil+ case "instance":+ instanceID, err := common.GetInstanceID(value.(string))+ if err != nil {+ return nil, errors.Errorf("invalid instance filter %q", value)+ }+ return qb.Q().Space("db.instance = ?", instanceID), nil+ case "environment":+ environment, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("failed to parse value %v to string", value)+ }+ if environment != "" {+ environmentID, err := common.GetEnvironmentID(environment)+ if err != nil {+ return nil, errors.Errorf("invalid environment filter %q", value)+ }+ return qb.Q().Space("COALESCE(db.environment, instance.environment) = ?", environmentID), nil+ }+ return qb.Q().Space("db.environment IS NULL AND instance.environment IS NULL"), nil+ case "engine":+ engineValue, ok := storepb.Engine_value[value.(string)]+ if !ok {+ return nil, errors.Errorf("invalid engine filter %q", value)+ }+ engine := storepb.Engine(engineValue)+ return qb.Q().Space("instance.metadata->>'engine' = ?", engine), nil+ case "name":+ return qb.Q().Space("db.name = ?", value), nil+ case "drifted":+ drifted, ok := value.(bool)+ if !ok {+ return nil, errors.Errorf("invalid drifted filter %q", value)+ }+ condition := "IS"+ if !drifted {+ condition = "IS NOT"+ }+ return qb.Q().Space(fmt.Sprintf("(db.metadata->>'drifted')::boolean %s TRUE", condition)), nil+ case "exclude_unassigned":+ if excludeUnassigned, ok := value.(bool); excludeUnassigned && ok {+ return qb.Q().Space("db.project != ?", common.DefaultProjectID), nil+ }+ return qb.Q().Space("TRUE"), nil+ case "table":+ return qb.Q().Space(`+ EXISTS (+ SELECT 1+ FROM json_array_elements(ds.metadata->'schemas') AS s,+ json_array_elements(s->'tables') AS t+ WHERE t->>'name' = ?+ )+ `, value.(string)), nil+ default:+ varStr, ok := variable.(string)+ if !ok {+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ if labelKey, ok := strings.CutPrefix(varStr, "labels."); ok {+ return parseToLabelFilterSQL("db.metadata", labelKey, value)+ }+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ }++ getFilter = func(expr celast.Expr) (*qb.Query, error) {+ q := qb.Q()+ switch expr.Kind() {+ case celast.CallKind:+ functionName := expr.AsCall().FunctionName()+ switch functionName {+ case celoperators.LogicalOr:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.Or("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.LogicalAnd:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.And("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.Equals:+ variable, value := getVariableAndValueFromExpr(expr)+ return parseToSQL(variable, value)+ case celoverloads.Matches:+ variable := expr.AsCall().Target().AsIdent()+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`invalid args for %q`, variable)+ }+ value := args[0].AsLiteral().Value()+ strValue, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)+ }+ strValue = strings.ToLower(strValue)++ switch variable {+ case "name":+ return qb.Q().Space("LOWER(db.name) LIKE ?", "%"+strValue+"%"), nil+ case "table":+ return qb.Q().Space(`EXISTS (+ SELECT 1+ FROM json_array_elements(ds.metadata->'schemas') AS s,+ json_array_elements(s->'tables') AS t+ WHERE t->>'name' LIKE ?`, "%"+strValue+"%"), nil+ default:+ return nil, errors.Errorf(`only "name" or "table" support %q operator, but found %q`, celoverloads.Matches, variable)+ }+ case celoperators.In:+ variable, value := getVariableAndValueFromExpr(expr)+ if variable == "engine" {+ return parseToEngineSQL(expr)+ } else if labelKey, ok := strings.CutPrefix(variable, "labels."); ok {+ return parseToLabelFilterSQL("db.metadata", labelKey, value)+ }+ return nil, errors.Errorf("unsupport variable %q", variable)+ case celoperators.LogicalNot:+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`only support !(engine in ["{engine1}", "{engine2}"]) format`)+ }+ qq, err := getFilter(args[0])+ if err != nil {+ return nil, err+ }+ return qb.Q().Space("(NOT (?))", qq), nil+ default:+ return nil, errors.Errorf("unexpected function %v", functionName)+ }+ default:+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())+ }+ }++ filterQ, err := getFilter(ast.NativeRep().Expr())+ if err != nil {+ return nil, err+ }+ return qb.Q().Space("(?)", filterQ), nil+}