Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: parse CEL filter in store with qb (#17828)
p0ny437cbafc1e37
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex 85b0c9c6b6..f2fb14ba1f 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -15,10 +15,6 @@ import ("connectrpc.com/connect""github.com/alexmullins/zip"- "github.com/google/cel-go/cel"- celast "github.com/google/cel-go/common/ast"- celoperators "github.com/google/cel-go/common/operators"- celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors""google.golang.org/protobuf/types/known/durationpb"@@ -1104,96 +1100,6 @@ func (s *SQLService) createQueryHistory(database *store.DatabaseMessage, queryTy}}-func getListQueryHistoryFilter(filter string) (*store.ListResourceFilter, error) {- if filter == "" {- return nil, nil- }-- e, err := cel.NewEnv()- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))- }- ast, iss := e.Parse(filter)- if iss != nil {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))- }-- var getFilter func(expr celast.Expr) (string, error)- var positionalArgs []any-- parseToSQL := func(variable, value any) (string, error) {- switch variable {- case "project":- projectID, err := common.GetProjectID(value.(string))- if err != nil {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid project filter %q", value))- }- positionalArgs = append(positionalArgs, projectID)- return fmt.Sprintf("query_history.project_id = $%d", len(positionalArgs)), nil- case "database":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("query_history.database = $%d", len(positionalArgs)), nil- case "instance":- positionalArgs = append(positionalArgs, value.(string))- return fmt.Sprintf("query_history.database LIKE $%d", len(positionalArgs)), nil- case "type":- historyType := store.QueryHistoryType(value.(string))- positionalArgs = append(positionalArgs, historyType)- return fmt.Sprintf("query_history.type = $%d", len(positionalArgs)), nil- case "statement":- positionalArgs = append(positionalArgs, value)- return fmt.Sprintf("query_history.statement LIKE $%d", len(positionalArgs)), nil- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))- }- }-- getFilter = func(expr celast.Expr) (string, error) {- switch expr.Kind() {- case celast.CallKind:- functionName := expr.AsCall().FunctionName()- switch functionName {- case celoperators.LogicalOr:- return getSubConditionFromExpr(expr, getFilter, "OR")- case celoperators.LogicalAnd:- return getSubConditionFromExpr(expr, getFilter, "AND")- case celoperators.Equals:- variable, value := getVariableAndValueFromExpr(expr)- return parseToSQL(variable, value)- case celoverloads.Matches:- variable := expr.AsCall().Target().AsIdent()- args := expr.AsCall().Args()- if len(args) != 1 {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))- }- value := args[0].AsLiteral().Value()- if variable != "statement" {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "statement" support %q operator, but found %q`, celoverloads.Matches, variable))- }- strValue, ok := value.(string)- if !ok {- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))- }- return parseToSQL("statement", "%"+strValue+"%")- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))- }- default:- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected expr kind %v", expr.Kind()))- }- }-- where, err := getFilter(ast.NativeRep().Expr())- if err != nil {- return nil, err- }-- return &store.ListResourceFilter{- Args: positionalArgs,- Where: "(" + where + ")",- }, nil-}-// SearchQueryHistories lists query histories.func (s *SQLService) SearchQueryHistories(ctx context.Context, req *connect.Request[v1pb.SearchQueryHistoriesRequest]) (*connect.Response[v1pb.SearchQueryHistoriesResponse], error) {request := req.Msg@@ -1217,11 +1123,11 @@ func (s *SQLService) SearchQueryHistories(ctx context.Context, req *connect.RequLimit: &limitPlusOne,Offset: &offset.offset,}- filter, err := getListQueryHistoryFilter(request.Filter)+ filterQ, err := store.GetListQueryHistoryFilter(request.Filter)if err != nil {- return nil, err+ return nil, connect.NewError(connect.CodeInvalidArgument, err)}- find.Filter = filter+ find.FilterQ = filterQhistoryList, err := s.store.ListQueryHistories(ctx, find)if err != nil {diff --git a/backend/store/filter.go b/backend/store/filter.gonew file mode 100644index 0000000000..96966a8cac--- /dev/null+++ b/backend/store/filter.go@@ -0,0 +1,36 @@+package store++import (+ "fmt"++ celast "github.com/google/cel-go/common/ast"+)++func getVariableAndValueFromExpr(expr celast.Expr) (string, any) {+ var variable string+ var value any+ for _, arg := range expr.AsCall().Args() {+ switch arg.Kind() {+ case celast.IdentKind:+ variable = arg.AsIdent()+ case celast.SelectKind:+ // Handle member selection like "labels.environment"+ sel := arg.AsSelect()+ if sel.Operand().Kind() == celast.IdentKind {+ variable = fmt.Sprintf("%s.%s", sel.Operand().AsIdent(), sel.FieldName())+ }+ case celast.LiteralKind:+ value = arg.AsLiteral().Value()+ case celast.ListKind:+ list := []any{}+ for _, e := range arg.AsList().Elements() {+ if e.Kind() == celast.LiteralKind {+ list = append(list, e.AsLiteral().Value())+ }+ }+ value = list+ default:+ }+ }+ return variable, value+}diff --git a/backend/store/query_history.go b/backend/store/query_history.goindex 27dad6d694..5bcc769455 100644--- a/backend/store/query_history.go+++ b/backend/store/query_history.go@@ -4,9 +4,12 @@ import ("context""time"- "google.golang.org/protobuf/encoding/protojson"-+ "github.com/google/cel-go/cel"+ celast "github.com/google/cel-go/common/ast"+ celoperators "github.com/google/cel-go/common/operators"+ celoverloads "github.com/google/cel-go/common/overloads""github.com/pkg/errors"+ "google.golang.org/protobuf/encoding/protojson""github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/qb"@@ -51,9 +54,9 @@ type FindQueryHistoryMessage struct {Database *stringType *QueryHistoryType- Limit *int- Offset *int- Filter *ListResourceFilter+ Limit *int+ Offset *int+ FilterQ *qb.Query}// CreateQueryHistory creates the query history.@@ -125,9 +128,8 @@ func (s *Store) ListQueryHistories(ctx context.Context, find *FindQueryHistoryMeWHERE TRUE`)- if filter := find.Filter; filter != nil {- // Convert $1, $2, etc. to ? for qb- q.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)+ if filterQ := find.FilterQ; filterQ != nil {+ q.And("?", filterQ)}if v := find.CreatorUID; v != nil {q.And("query_history.creator_id = ?", *v)@@ -204,3 +206,98 @@ func (s *Store) ListQueryHistories(ctx context.Context, find *FindQueryHistoryMereturn queryHistories, nil}++func GetListQueryHistoryFilter(filter string) (*qb.Query, error) {+ if filter == "" {+ return nil, nil+ }++ e, err := cel.NewEnv()+ if err != nil {+ return nil, errors.Errorf("failed to create cel env")+ }+ ast, iss := e.Parse(filter)+ if iss != nil {+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())+ }++ var getFilter func(expr celast.Expr) (*qb.Query, error)++ parseToSQL := func(variable, value any) (*qb.Query, error) {+ switch variable {+ case "project":+ projectID, err := common.GetProjectID(value.(string))+ if err != nil {+ return nil, errors.Errorf("invalid project filter %q", value)+ }+ return qb.Q().Space("query_history.project_id = ?", projectID), nil+ case "database":+ return qb.Q().Space("query_history.database = ?", value.(string)), nil+ case "instance":+ return qb.Q().Space("query_history.database LIKE ?", value.(string)), nil+ case "type":+ historyType := QueryHistoryType(value.(string))+ return qb.Q().Space("query_history.type = ?", historyType), nil+ case "statement":+ return qb.Q().Space("query_history.statement LIKE ?", value), nil+ default:+ return nil, errors.Errorf("unsupport variable %q", variable)+ }+ }++ getFilter = func(expr celast.Expr) (*qb.Query, error) {+ q := qb.Q()+ switch expr.Kind() {+ case celast.CallKind:+ functionName := expr.AsCall().FunctionName()+ switch functionName {+ case celoperators.LogicalOr:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.Or("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.LogicalAnd:+ for _, arg := range expr.AsCall().Args() {+ qq, err := getFilter(arg)+ if err != nil {+ return nil, err+ }+ q.And("?", qq)+ }+ return qb.Q().Space("(?)", q), nil+ case celoperators.Equals:+ variable, value := getVariableAndValueFromExpr(expr)+ return parseToSQL(variable, value)+ case celoverloads.Matches:+ variable := expr.AsCall().Target().AsIdent()+ args := expr.AsCall().Args()+ if len(args) != 1 {+ return nil, errors.Errorf(`invalid args for %q`, variable)+ }+ value := args[0].AsLiteral().Value()+ if variable != "statement" {+ return nil, errors.Errorf(`only "statement" support %q operator, but found %q`, celoverloads.Matches, variable)+ }+ strValue, ok := value.(string)+ if !ok {+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)+ }+ return parseToSQL("statement", "%"+strValue+"%")+ default:+ return nil, errors.Errorf("unexpected function %v", functionName)+ }+ default:+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())+ }+ }++ q, err := getFilter(ast.NativeRep().Expr())+ if err != nil {+ return nil, err+ }+ return qb.Q().Space("(?)", q), nil+}diff --git a/backend/store/query_history_filter_test.go b/backend/store/query_history_filter_test.gonew file mode 100644index 0000000000..fed50dda74--- /dev/null+++ b/backend/store/query_history_filter_test.go@@ -0,0 +1,303 @@+package store++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestGetListQueryHistoryFilter(t *testing.T) {+ tests := []struct {+ name string+ filter string+ wantSQL string+ wantArgs []any+ wantErr bool+ errContains string+ }{+ {+ name: "empty filter",+ filter: "",+ wantSQL: "",+ wantArgs: nil,+ wantErr: false,+ },+ {+ name: "project filter",+ filter: `project == "projects/test-project"`,+ wantSQL: "(query_history.project_id = $1)",+ wantArgs: []any{"test-project"},+ wantErr: false,+ },+ {+ name: "database filter",+ filter: `database == "instances/test-instance/databases/test-db"`,+ wantSQL: "(query_history.database = $1)",+ wantArgs: []any{"instances/test-instance/databases/test-db"},+ wantErr: false,+ },+ {+ name: "instance filter",+ filter: `instance == "instances/test-instance/%"`,+ wantSQL: "(query_history.database LIKE $1)",+ wantArgs: []any{"instances/test-instance/%"},+ wantErr: false,+ },+ {+ name: "type filter",+ filter: `type == "QUERY"`,+ wantSQL: "(query_history.type = $1)",+ wantArgs: []any{QueryHistoryType("QUERY")},+ wantErr: false,+ },+ {+ name: "statement exact match",+ filter: `statement == "SELECT * FROM users"`,+ wantSQL: "(query_history.statement LIKE $1)",+ wantArgs: []any{"SELECT * FROM users"},+ wantErr: false,+ },+ {+ name: "statement matches operator",+ filter: `statement.matches("SELECT")`,+ wantSQL: "(query_history.statement LIKE $1)",+ wantArgs: []any{"%SELECT%"},+ wantErr: false,+ },+ {+ name: "AND condition with project and database",+ filter: `project == "projects/test-project" && database == "instances/test-instance/databases/test-db"`,+ wantSQL: "((query_history.project_id = $1 AND query_history.database = $2))",+ wantArgs: []any{"test-project", "instances/test-instance/databases/test-db"},+ wantErr: false,+ },+ {+ name: "AND condition with three filters",+ filter: `project == "projects/test-project" && database == "instances/test-instance/databases/test-db" && type == "QUERY"`,+ wantSQL: "(((query_history.project_id = $1 AND query_history.database = $2) AND query_history.type = $3))",+ wantArgs: []any{"test-project", "instances/test-instance/databases/test-db", QueryHistoryType("QUERY")},+ wantErr: false,+ },+ {+ name: "OR condition with project and database",+ filter: `project == "projects/project1" || project == "projects/project2"`,+ wantSQL: "((query_history.project_id = $1 OR query_history.project_id = $2))",+ wantArgs: []any{"project1", "project2"},+ wantErr: false,+ },+ {+ name: "OR condition with type",+ filter: `type == "QUERY" || type == "EXPORT"`,+ wantSQL: "((query_history.type = $1 OR query_history.type = $2))",+ wantArgs: []any{QueryHistoryType("QUERY"), QueryHistoryType("EXPORT")},+ wantErr: false,+ },+ {+ name: "complex nested AND/OR",+ filter: `(project == "projects/p1" || project == "projects/p2") && type == "QUERY"`,+ wantSQL: "(((query_history.project_id = $1 OR query_history.project_id = $2) AND query_history.type = $3))",+ wantArgs: []any{"p1", "p2", QueryHistoryType("QUERY")},+ wantErr: false,+ },+ {+ name: "complex nested with statement matches",+ filter: `project == "projects/test" && statement.matches("SELECT") && type == "QUERY"`,+ wantSQL: "(((query_history.project_id = $1 AND query_history.statement LIKE $2) AND query_history.type = $3))",+ wantArgs: []any{"test", "%SELECT%", QueryHistoryType("QUERY")},+ wantErr: false,+ },+ {+ name: "invalid filter syntax",+ filter: `invalid syntax {{`,+ wantErr: true,+ errContains: "failed to parse filter",+ },+ {+ name: "unsupported variable",+ filter: `unsupported == "value"`,+ wantErr: true,+ errContains: "unsupport variable",+ },+ {+ name: "matches on non-statement variable",+ filter: `project.matches("test")`,+ wantErr: true,+ errContains: `only "statement" support`,+ },+ {+ name: "invalid project format",+ filter: `project == "invalid-format"`,+ wantErr: true,+ errContains: "invalid project filter",+ },+ {+ name: "matches with non-string value",+ filter: `statement.matches(123)`,+ wantErr: true,+ errContains: "expect string",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ q, err := GetListQueryHistoryFilter(tt.filter)++ if tt.wantErr {+ require.Error(t, err)+ if tt.errContains != "" {+ require.ErrorContains(t, err, tt.errContains)+ }+ return+ }++ require.NoError(t, err)++ if tt.filter == "" {+ require.Nil(t, q)+ return+ }++ require.NotNil(t, q)++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, tt.wantSQL, sql)+ require.Equal(t, tt.wantArgs, args)+ })+ }+}++func TestGetListQueryHistoryFilter_EdgeCases(t *testing.T) {+ tests := []struct {+ name string+ filter string+ description string+ wantSQL string+ wantArgs []any+ wantErr bool+ errContains string+ }{+ {+ name: "statement with special characters",+ filter: `statement == "SELECT * FROM users WHERE name = 'test'"`,+ description: "statement with quotes should be handled correctly",+ wantSQL: "(query_history.statement LIKE $1)",+ wantArgs: []any{"SELECT * FROM users WHERE name = 'test'"},+ wantErr: false,+ },+ {+ name: "statement matches with wildcard pattern",+ filter: `statement.matches("SELECT.*FROM")`,+ description: "regex pattern in matches should be wrapped with %",+ wantSQL: "(query_history.statement LIKE $1)",+ wantArgs: []any{"%SELECT.*FROM%"},+ wantErr: false,+ },+ {+ name: "multiple OR conditions",+ filter: `project == "projects/p1" || project == "projects/p2" || project == "projects/p3"`,+ description: "multiple OR conditions should be chained",+ wantSQL: "(((query_history.project_id = $1 OR query_history.project_id = $2) OR query_history.project_id = $3))",+ wantArgs: []any{"p1", "p2", "p3"},+ wantErr: false,+ },+ {+ name: "database with instance prefix",+ filter: `database == "instances/prod-instance/databases/users_db"`,+ description: "full database path should be preserved",+ wantSQL: "(query_history.database = $1)",+ wantArgs: []any{"instances/prod-instance/databases/users_db"},+ wantErr: false,+ },+ {+ name: "instance with wildcard",+ filter: `instance == "instances/prod-%"`,+ description: "instance filter with wildcard for LIKE query",+ wantSQL: "(query_history.database LIKE $1)",+ wantArgs: []any{"instances/prod-%"},+ wantErr: false,+ },+ {+ name: "complex filter with all supported fields",+ filter: `project == "projects/test" && database == "instances/i1/databases/db1" && type == "EXPORT" && statement.matches("INSERT")`,+ description: "combination of all supported filter types",+ wantSQL: "(((query_history.project_id = $1 AND query_history.database = $2) AND (query_history.type = $3 AND query_history.statement LIKE $4)))",+ wantArgs: []any{"test", "instances/i1/databases/db1", QueryHistoryType("EXPORT"), "%INSERT%"},+ wantErr: false,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ q, err := GetListQueryHistoryFilter(tt.filter)++ if tt.wantErr {+ require.Error(t, err, tt.description)+ if tt.errContains != "" {+ require.ErrorContains(t, err, tt.errContains)+ }+ return+ }++ require.NoError(t, err, tt.description)+ require.NotNil(t, q)++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, tt.wantSQL, sql, tt.description)+ require.Equal(t, tt.wantArgs, args, tt.description)+ })+ }+}++// TestGetListQueryHistoryFilter_CompareWithOriginal verifies that the new implementation+// produces equivalent SQL to the original v1 implementation+func TestGetListQueryHistoryFilter_CompareWithOriginal(t *testing.T) {+ tests := []struct {+ name string+ filter string+ expectedConditions []string // The conditions we expect in the WHERE clause+ }{+ {+ name: "simple project filter",+ filter: `project == "projects/test-project"`,+ expectedConditions: []string{"query_history.project_id = $1"},+ },+ {+ name: "database filter",+ filter: `database == "instances/test-instance/databases/test-db"`,+ expectedConditions: []string{"query_history.database = $1"},+ },+ {+ name: "type and project combined",+ filter: `type == "QUERY" && project == "projects/my-project"`,+ expectedConditions: []string{"query_history.type = $1", "AND", "query_history.project_id = $2"},+ },+ {+ name: "statement matches",+ filter: `statement.matches("SELECT")`,+ expectedConditions: []string{"query_history.statement LIKE $1"},+ },+ {+ name: "OR condition",+ filter: `type == "QUERY" || type == "EXPORT"`,+ expectedConditions: []string{"query_history.type = $1", "OR", "query_history.type = $2"},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ q, err := GetListQueryHistoryFilter(tt.filter)+ require.NoError(t, err)+ require.NotNil(t, q)++ sql, _, err := q.ToSQL()+ require.NoError(t, err)++ // Verify that all expected conditions are present in the SQL+ for _, condition := range tt.expectedConditions {+ require.Contains(t, sql, condition, "SQL should contain expected condition")+ }+ })+ }+}