Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: parse CEL filter in store with qb (#17828)

p0ny437cbafc1e37
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index 85b0c9c6b6..f2fb14ba1f 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -15,10 +15,6 @@ import (
"connectrpc.com/connect"
"github.com/alexmullins/zip"
- "github.com/google/cel-go/cel"
- celast "github.com/google/cel-go/common/ast"
- celoperators "github.com/google/cel-go/common/operators"
- celoverloads "github.com/google/cel-go/common/overloads"
"github.com/pkg/errors"
"google.golang.org/protobuf/types/known/durationpb"
@@ -1104,96 +1100,6 @@ func (s *SQLService) createQueryHistory(database *store.DatabaseMessage, queryTy
}
}
-func getListQueryHistoryFilter(filter string) (*store.ListResourceFilter, error) {
- if filter == "" {
- return nil, nil
- }
-
- e, err := cel.NewEnv()
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create cel env"))
- }
- ast, iss := e.Parse(filter)
- if iss != nil {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String()))
- }
-
- var getFilter func(expr celast.Expr) (string, error)
- var positionalArgs []any
-
- parseToSQL := func(variable, value any) (string, error) {
- switch variable {
- case "project":
- projectID, err := common.GetProjectID(value.(string))
- if err != nil {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid project filter %q", value))
- }
- positionalArgs = append(positionalArgs, projectID)
- return fmt.Sprintf("query_history.project_id = $%d", len(positionalArgs)), nil
- case "database":
- positionalArgs = append(positionalArgs, value.(string))
- return fmt.Sprintf("query_history.database = $%d", len(positionalArgs)), nil
- case "instance":
- positionalArgs = append(positionalArgs, value.(string))
- return fmt.Sprintf("query_history.database LIKE $%d", len(positionalArgs)), nil
- case "type":
- historyType := store.QueryHistoryType(value.(string))
- positionalArgs = append(positionalArgs, historyType)
- return fmt.Sprintf("query_history.type = $%d", len(positionalArgs)), nil
- case "statement":
- positionalArgs = append(positionalArgs, value)
- return fmt.Sprintf("query_history.statement LIKE $%d", len(positionalArgs)), nil
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %q", variable))
- }
- }
-
- getFilter = func(expr celast.Expr) (string, error) {
- switch expr.Kind() {
- case celast.CallKind:
- functionName := expr.AsCall().FunctionName()
- switch functionName {
- case celoperators.LogicalOr:
- return getSubConditionFromExpr(expr, getFilter, "OR")
- case celoperators.LogicalAnd:
- return getSubConditionFromExpr(expr, getFilter, "AND")
- case celoperators.Equals:
- variable, value := getVariableAndValueFromExpr(expr)
- return parseToSQL(variable, value)
- case celoverloads.Matches:
- variable := expr.AsCall().Target().AsIdent()
- args := expr.AsCall().Args()
- if len(args) != 1 {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`invalid args for %q`, variable))
- }
- value := args[0].AsLiteral().Value()
- if variable != "statement" {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`only "statement" support %q operator, but found %q`, celoverloads.Matches, variable))
- }
- strValue, ok := value.(string)
- if !ok {
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", value))
- }
- return parseToSQL("statement", "%"+strValue+"%")
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))
- }
- default:
- return "", connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected expr kind %v", expr.Kind()))
- }
- }
-
- where, err := getFilter(ast.NativeRep().Expr())
- if err != nil {
- return nil, err
- }
-
- return &store.ListResourceFilter{
- Args: positionalArgs,
- Where: "(" + where + ")",
- }, nil
-}
-
// SearchQueryHistories lists query histories.
func (s *SQLService) SearchQueryHistories(ctx context.Context, req *connect.Request[v1pb.SearchQueryHistoriesRequest]) (*connect.Response[v1pb.SearchQueryHistoriesResponse], error) {
request := req.Msg
@@ -1217,11 +1123,11 @@ func (s *SQLService) SearchQueryHistories(ctx context.Context, req *connect.Requ
Limit: &limitPlusOne,
Offset: &offset.offset,
}
- filter, err := getListQueryHistoryFilter(request.Filter)
+ filterQ, err := store.GetListQueryHistoryFilter(request.Filter)
if err != nil {
- return nil, err
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
}
- find.Filter = filter
+ find.FilterQ = filterQ
historyList, err := s.store.ListQueryHistories(ctx, find)
if err != nil {
diff --git a/backend/store/filter.go b/backend/store/filter.go
new file mode 100644
index 0000000000..96966a8cac
--- /dev/null
+++ b/backend/store/filter.go
@@ -0,0 +1,36 @@
+package store
+
+import (
+ "fmt"
+
+ celast "github.com/google/cel-go/common/ast"
+)
+
+func getVariableAndValueFromExpr(expr celast.Expr) (string, any) {
+ var variable string
+ var value any
+ for _, arg := range expr.AsCall().Args() {
+ switch arg.Kind() {
+ case celast.IdentKind:
+ variable = arg.AsIdent()
+ case celast.SelectKind:
+ // Handle member selection like "labels.environment"
+ sel := arg.AsSelect()
+ if sel.Operand().Kind() == celast.IdentKind {
+ variable = fmt.Sprintf("%s.%s", sel.Operand().AsIdent(), sel.FieldName())
+ }
+ case celast.LiteralKind:
+ value = arg.AsLiteral().Value()
+ case celast.ListKind:
+ list := []any{}
+ for _, e := range arg.AsList().Elements() {
+ if e.Kind() == celast.LiteralKind {
+ list = append(list, e.AsLiteral().Value())
+ }
+ }
+ value = list
+ default:
+ }
+ }
+ return variable, value
+}
diff --git a/backend/store/query_history.go b/backend/store/query_history.go
index 27dad6d694..5bcc769455 100644
--- a/backend/store/query_history.go
+++ b/backend/store/query_history.go
@@ -4,9 +4,12 @@ import (
"context"
"time"
- "google.golang.org/protobuf/encoding/protojson"
-
+ "github.com/google/cel-go/cel"
+ celast "github.com/google/cel-go/common/ast"
+ celoperators "github.com/google/cel-go/common/operators"
+ celoverloads "github.com/google/cel-go/common/overloads"
"github.com/pkg/errors"
+ "google.golang.org/protobuf/encoding/protojson"
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/qb"
@@ -51,9 +54,9 @@ type FindQueryHistoryMessage struct {
Database *string
Type *QueryHistoryType
- Limit *int
- Offset *int
- Filter *ListResourceFilter
+ Limit *int
+ Offset *int
+ FilterQ *qb.Query
}
// CreateQueryHistory creates the query history.
@@ -125,9 +128,8 @@ func (s *Store) ListQueryHistories(ctx context.Context, find *FindQueryHistoryMe
WHERE TRUE
`)
- if filter := find.Filter; filter != nil {
- // Convert $1, $2, etc. to ? for qb
- q.And(ConvertDollarPlaceholders(filter.Where), filter.Args...)
+ if filterQ := find.FilterQ; filterQ != nil {
+ q.And("?", filterQ)
}
if v := find.CreatorUID; v != nil {
q.And("query_history.creator_id = ?", *v)
@@ -204,3 +206,98 @@ func (s *Store) ListQueryHistories(ctx context.Context, find *FindQueryHistoryMe
return queryHistories, nil
}
+
+func GetListQueryHistoryFilter(filter string) (*qb.Query, error) {
+ if filter == "" {
+ return nil, nil
+ }
+
+ e, err := cel.NewEnv()
+ if err != nil {
+ return nil, errors.Errorf("failed to create cel env")
+ }
+ ast, iss := e.Parse(filter)
+ if iss != nil {
+ return nil, errors.Errorf("failed to parse filter %v, error: %v", filter, iss.String())
+ }
+
+ var getFilter func(expr celast.Expr) (*qb.Query, error)
+
+ parseToSQL := func(variable, value any) (*qb.Query, error) {
+ switch variable {
+ case "project":
+ projectID, err := common.GetProjectID(value.(string))
+ if err != nil {
+ return nil, errors.Errorf("invalid project filter %q", value)
+ }
+ return qb.Q().Space("query_history.project_id = ?", projectID), nil
+ case "database":
+ return qb.Q().Space("query_history.database = ?", value.(string)), nil
+ case "instance":
+ return qb.Q().Space("query_history.database LIKE ?", value.(string)), nil
+ case "type":
+ historyType := QueryHistoryType(value.(string))
+ return qb.Q().Space("query_history.type = ?", historyType), nil
+ case "statement":
+ return qb.Q().Space("query_history.statement LIKE ?", value), nil
+ default:
+ return nil, errors.Errorf("unsupport variable %q", variable)
+ }
+ }
+
+ getFilter = func(expr celast.Expr) (*qb.Query, error) {
+ q := qb.Q()
+ switch expr.Kind() {
+ case celast.CallKind:
+ functionName := expr.AsCall().FunctionName()
+ switch functionName {
+ case celoperators.LogicalOr:
+ for _, arg := range expr.AsCall().Args() {
+ qq, err := getFilter(arg)
+ if err != nil {
+ return nil, err
+ }
+ q.Or("?", qq)
+ }
+ return qb.Q().Space("(?)", q), nil
+ case celoperators.LogicalAnd:
+ for _, arg := range expr.AsCall().Args() {
+ qq, err := getFilter(arg)
+ if err != nil {
+ return nil, err
+ }
+ q.And("?", qq)
+ }
+ return qb.Q().Space("(?)", q), nil
+ case celoperators.Equals:
+ variable, value := getVariableAndValueFromExpr(expr)
+ return parseToSQL(variable, value)
+ case celoverloads.Matches:
+ variable := expr.AsCall().Target().AsIdent()
+ args := expr.AsCall().Args()
+ if len(args) != 1 {
+ return nil, errors.Errorf(`invalid args for %q`, variable)
+ }
+ value := args[0].AsLiteral().Value()
+ if variable != "statement" {
+ return nil, errors.Errorf(`only "statement" support %q operator, but found %q`, celoverloads.Matches, variable)
+ }
+ strValue, ok := value.(string)
+ if !ok {
+ return nil, errors.Errorf("expect string, got %T, hint: filter literals should be string", value)
+ }
+ return parseToSQL("statement", "%"+strValue+"%")
+ default:
+ return nil, errors.Errorf("unexpected function %v", functionName)
+ }
+ default:
+ return nil, errors.Errorf("unexpected expr kind %v", expr.Kind())
+ }
+ }
+
+ q, err := getFilter(ast.NativeRep().Expr())
+ if err != nil {
+ return nil, err
+ }
+ return qb.Q().Space("(?)", q), nil
+}
diff --git a/backend/store/query_history_filter_test.go b/backend/store/query_history_filter_test.go
new file mode 100644
index 0000000000..fed50dda74
--- /dev/null
+++ b/backend/store/query_history_filter_test.go
@@ -0,0 +1,303 @@
+package store
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetListQueryHistoryFilter(t *testing.T) {
+ tests := []struct {
+ name string
+ filter string
+ wantSQL string
+ wantArgs []any
+ wantErr bool
+ errContains string
+ }{
+ {
+ name: "empty filter",
+ filter: "",
+ wantSQL: "",
+ wantArgs: nil,
+ wantErr: false,
+ },
+ {
+ name: "project filter",
+ filter: `project == "projects/test-project"`,
+ wantSQL: "(query_history.project_id = $1)",
+ wantArgs: []any{"test-project"},
+ wantErr: false,
+ },
+ {
+ name: "database filter",
+ filter: `database == "instances/test-instance/databases/test-db"`,
+ wantSQL: "(query_history.database = $1)",
+ wantArgs: []any{"instances/test-instance/databases/test-db"},
+ wantErr: false,
+ },
+ {
+ name: "instance filter",
+ filter: `instance == "instances/test-instance/%"`,
+ wantSQL: "(query_history.database LIKE $1)",
+ wantArgs: []any{"instances/test-instance/%"},
+ wantErr: false,
+ },
+ {
+ name: "type filter",
+ filter: `type == "QUERY"`,
+ wantSQL: "(query_history.type = $1)",
+ wantArgs: []any{QueryHistoryType("QUERY")},
+ wantErr: false,
+ },
+ {
+ name: "statement exact match",
+ filter: `statement == "SELECT * FROM users"`,
+ wantSQL: "(query_history.statement LIKE $1)",
+ wantArgs: []any{"SELECT * FROM users"},
+ wantErr: false,
+ },
+ {
+ name: "statement matches operator",
+ filter: `statement.matches("SELECT")`,
+ wantSQL: "(query_history.statement LIKE $1)",
+ wantArgs: []any{"%SELECT%"},
+ wantErr: false,
+ },
+ {
+ name: "AND condition with project and database",
+ filter: `project == "projects/test-project" && database == "instances/test-instance/databases/test-db"`,
+ wantSQL: "((query_history.project_id = $1 AND query_history.database = $2))",
+ wantArgs: []any{"test-project", "instances/test-instance/databases/test-db"},
+ wantErr: false,
+ },
+ {
+ name: "AND condition with three filters",
+ filter: `project == "projects/test-project" && database == "instances/test-instance/databases/test-db" && type == "QUERY"`,
+ wantSQL: "(((query_history.project_id = $1 AND query_history.database = $2) AND query_history.type = $3))",
+ wantArgs: []any{"test-project", "instances/test-instance/databases/test-db", QueryHistoryType("QUERY")},
+ wantErr: false,
+ },
+ {
+ name: "OR condition with project and database",
+ filter: `project == "projects/project1" || project == "projects/project2"`,
+ wantSQL: "((query_history.project_id = $1 OR query_history.project_id = $2))",
+ wantArgs: []any{"project1", "project2"},
+ wantErr: false,
+ },
+ {
+ name: "OR condition with type",
+ filter: `type == "QUERY" || type == "EXPORT"`,
+ wantSQL: "((query_history.type = $1 OR query_history.type = $2))",
+ wantArgs: []any{QueryHistoryType("QUERY"), QueryHistoryType("EXPORT")},
+ wantErr: false,
+ },
+ {
+ name: "complex nested AND/OR",
+ filter: `(project == "projects/p1" || project == "projects/p2") && type == "QUERY"`,
+ wantSQL: "(((query_history.project_id = $1 OR query_history.project_id = $2) AND query_history.type = $3))",
+ wantArgs: []any{"p1", "p2", QueryHistoryType("QUERY")},
+ wantErr: false,
+ },
+ {
+ name: "complex nested with statement matches",
+ filter: `project == "projects/test" && statement.matches("SELECT") && type == "QUERY"`,
+ wantSQL: "(((query_history.project_id = $1 AND query_history.statement LIKE $2) AND query_history.type = $3))",
+ wantArgs: []any{"test", "%SELECT%", QueryHistoryType("QUERY")},
+ wantErr: false,
+ },
+ {
+ name: "invalid filter syntax",
+ filter: `invalid syntax {{`,
+ wantErr: true,
+ errContains: "failed to parse filter",
+ },
+ {
+ name: "unsupported variable",
+ filter: `unsupported == "value"`,
+ wantErr: true,
+ errContains: "unsupport variable",
+ },
+ {
+ name: "matches on non-statement variable",
+ filter: `project.matches("test")`,
+ wantErr: true,
+ errContains: `only "statement" support`,
+ },
+ {
+ name: "invalid project format",
+ filter: `project == "invalid-format"`,
+ wantErr: true,
+ errContains: "invalid project filter",
+ },
+ {
+ name: "matches with non-string value",
+ filter: `statement.matches(123)`,
+ wantErr: true,
+ errContains: "expect string",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ q, err := GetListQueryHistoryFilter(tt.filter)
+
+ if tt.wantErr {
+ require.Error(t, err)
+ if tt.errContains != "" {
+ require.ErrorContains(t, err, tt.errContains)
+ }
+ return
+ }
+
+ require.NoError(t, err)
+
+ if tt.filter == "" {
+ require.Nil(t, q)
+ return
+ }
+
+ require.NotNil(t, q)
+
+ sql, args, err := q.ToSQL()
+ require.NoError(t, err)
+ require.Equal(t, tt.wantSQL, sql)
+ require.Equal(t, tt.wantArgs, args)
+ })
+ }
+}
+
+func TestGetListQueryHistoryFilter_EdgeCases(t *testing.T) {
+ tests := []struct {
+ name string
+ filter string
+ description string
+ wantSQL string
+ wantArgs []any
+ wantErr bool
+ errContains string
+ }{
+ {
+ name: "statement with special characters",
+ filter: `statement == "SELECT * FROM users WHERE name = 'test'"`,
+ description: "statement with quotes should be handled correctly",
+ wantSQL: "(query_history.statement LIKE $1)",
+ wantArgs: []any{"SELECT * FROM users WHERE name = 'test'"},
+ wantErr: false,
+ },
+ {
+ name: "statement matches with wildcard pattern",
+ filter: `statement.matches("SELECT.*FROM")`,
+ description: "regex pattern in matches should be wrapped with %",
+ wantSQL: "(query_history.statement LIKE $1)",
+ wantArgs: []any{"%SELECT.*FROM%"},
+ wantErr: false,
+ },
+ {
+ name: "multiple OR conditions",
+ filter: `project == "projects/p1" || project == "projects/p2" || project == "projects/p3"`,
+ description: "multiple OR conditions should be chained",
+ wantSQL: "(((query_history.project_id = $1 OR query_history.project_id = $2) OR query_history.project_id = $3))",
+ wantArgs: []any{"p1", "p2", "p3"},
+ wantErr: false,
+ },
+ {
+ name: "database with instance prefix",
+ filter: `database == "instances/prod-instance/databases/users_db"`,
+ description: "full database path should be preserved",
+ wantSQL: "(query_history.database = $1)",
+ wantArgs: []any{"instances/prod-instance/databases/users_db"},
+ wantErr: false,
+ },
+ {
+ name: "instance with wildcard",
+ filter: `instance == "instances/prod-%"`,
+ description: "instance filter with wildcard for LIKE query",
+ wantSQL: "(query_history.database LIKE $1)",
+ wantArgs: []any{"instances/prod-%"},
+ wantErr: false,
+ },
+ {
+ name: "complex filter with all supported fields",
+ filter: `project == "projects/test" && database == "instances/i1/databases/db1" && type == "EXPORT" && statement.matches("INSERT")`,
+ description: "combination of all supported filter types",
+ wantSQL: "(((query_history.project_id = $1 AND query_history.database = $2) AND (query_history.type = $3 AND query_history.statement LIKE $4)))",
+ wantArgs: []any{"test", "instances/i1/databases/db1", QueryHistoryType("EXPORT"), "%INSERT%"},
+ wantErr: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ q, err := GetListQueryHistoryFilter(tt.filter)
+
+ if tt.wantErr {
+ require.Error(t, err, tt.description)
+ if tt.errContains != "" {
+ require.ErrorContains(t, err, tt.errContains)
+ }
+ return
+ }
+
+ require.NoError(t, err, tt.description)
+ require.NotNil(t, q)
+
+ sql, args, err := q.ToSQL()
+ require.NoError(t, err)
+ require.Equal(t, tt.wantSQL, sql, tt.description)
+ require.Equal(t, tt.wantArgs, args, tt.description)
+ })
+ }
+}
+
+// TestGetListQueryHistoryFilter_CompareWithOriginal verifies that the new implementation
+// produces equivalent SQL to the original v1 implementation
+func TestGetListQueryHistoryFilter_CompareWithOriginal(t *testing.T) {
+ tests := []struct {
+ name string
+ filter string
+ expectedConditions []string // The conditions we expect in the WHERE clause
+ }{
+ {
+ name: "simple project filter",
+ filter: `project == "projects/test-project"`,
+ expectedConditions: []string{"query_history.project_id = $1"},
+ },
+ {
+ name: "database filter",
+ filter: `database == "instances/test-instance/databases/test-db"`,
+ expectedConditions: []string{"query_history.database = $1"},
+ },
+ {
+ name: "type and project combined",
+ filter: `type == "QUERY" && project == "projects/my-project"`,
+ expectedConditions: []string{"query_history.type = $1", "AND", "query_history.project_id = $2"},
+ },
+ {
+ name: "statement matches",
+ filter: `statement.matches("SELECT")`,
+ expectedConditions: []string{"query_history.statement LIKE $1"},
+ },
+ {
+ name: "OR condition",
+ filter: `type == "QUERY" || type == "EXPORT"`,
+ expectedConditions: []string{"query_history.type = $1", "OR", "query_history.type = $2"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ q, err := GetListQueryHistoryFilter(tt.filter)
+ require.NoError(t, err)
+ require.NotNil(t, q)
+
+ sql, _, err := q.ToSQL()
+ require.NoError(t, err)
+
+ // Verify that all expected conditions are present in the SQL
+ for _, condition := range tt.expectedConditions {
+ require.Contains(t, sql, condition, "SQL should contain expected condition")
+ }
+ })
+ }
+}