Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(cosmosdb): migrate parser from ANTLR to omni (#19781)

h3n4l5b55b5badbba
diff --git a/CLAUDE.md b/CLAUDE.md
index 8f5db1809a..37bbf27495 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -20,6 +20,7 @@ This file provides guidance to AI coding assistants when working with code in th
3. **Auto-fix** — Use `golangci-lint run --fix --allow-parallel-runners` to fix issues automatically
4. **Test** — Run relevant tests before committing
5. **Build** — `go build -ldflags "-w -s" -p=16 -o ./bytebase-build/bytebase ./backend/bin/server/main.go`
+6. **Tidy** — After upgrading Go dependencies, run `go mod tidy` to clean up `go.mod` and `go.sum`
### Frontend Code Changes
diff --git a/backend/plugin/parser/base/ast.go b/backend/plugin/parser/base/ast.go
index 89bf5ab175..1e233597fc 100644
--- a/backend/plugin/parser/base/ast.go
+++ b/backend/plugin/parser/base/ast.go
@@ -18,7 +18,7 @@ type AST interface {
// ANTLRAST is the AST implementation for ANTLR-based parsers.
// Supported engines: PostgreSQL, MySQL, MariaDB, OceanBase, MSSQL, Oracle, Redshift,
-// Snowflake, BigQuery, Spanner, Doris, Cassandra, Trino, PartiQL, CosmosDB.
+// Snowflake, BigQuery, Spanner, Doris, Cassandra, Trino, PartiQL.
//
// Parser packages can use this directly or embed it to add engine-specific fields.
type ANTLRAST struct {
diff --git a/backend/plugin/parser/cosmosdb/cosmosdb.go b/backend/plugin/parser/cosmosdb/cosmosdb.go
index d4bb0771f3..8c6ba17238 100644
--- a/backend/plugin/parser/cosmosdb/cosmosdb.go
+++ b/backend/plugin/parser/cosmosdb/cosmosdb.go
@@ -1,80 +1,83 @@
package cosmosdb
import (
- "strings"
+ "unicode/utf8"
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/cosmosdb"
+ omnicosmosdb "github.com/bytebase/omni/cosmosdb"
+ "github.com/bytebase/omni/cosmosdb/ast"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
- "github.com/bytebase/bytebase/backend/utils"
)
-// ParseCosmosDBQuery parses the given CosmosDB SQL statement by using antlr4. Returns a list of AST and token stream if no error.
-// Note: CosmosDB only supports single SELECT statements, so this will typically return a list with one element.
-func ParseCosmosDBQuery(statement string) ([]*base.ANTLRAST, error) {
- stmts, err := SplitSQL(statement)
+// OmniAST wraps an omni CosmosDB AST node and implements the base.AST interface.
+type OmniAST struct {
+ Node ast.Node
+ Text string
+ StartPosition *storepb.Position
+}
+
+// ASTStartPosition implements base.AST.
+func (a *OmniAST) ASTStartPosition() *storepb.Position {
+ return a.StartPosition
+}
+
+// GetOmniNode extracts the omni AST node from a base.AST interface.
+func GetOmniNode(a base.AST) (ast.Node, bool) {
+ if a == nil {
+ return nil, false
+ }
+ omniAST, ok := a.(*OmniAST)
+ if !ok {
+ return nil, false
+ }
+ return omniAST.Node, true
+}
+
+// ParseCosmosDB parses the given CosmosDB SQL statement using omni's parser.
+func ParseCosmosDB(statement string) ([]*OmniAST, error) {
+ stmts, err := omnicosmosdb.Parse(statement)
if err != nil {
return nil, err
}
- var result []*base.ANTLRAST
+ var result []*OmniAST
for _, stmt := range stmts {
- if stmt.Empty {
+ if stmt.Empty() {
continue
}
-
- parseResult, err := parseSingleCosmosDBQuery(stmt.Text, stmt.BaseLine())
- if err != nil {
- return nil, err
- }
- result = append(result, parseResult)
+ result = append(result, &OmniAST{
+ Node: stmt.AST,
+ Text: stmt.Text,
+ StartPosition: byteOffsetToRunePosition(statement, stmt.ByteStart),
+ })
}
-
return result, nil
}
-func parseSingleCosmosDBQuery(statement string, baseLine int) (*base.ANTLRAST, error) {
- statement = strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon)
- inputStream := antlr.NewInputStream(statement)
- lexer := parser.NewCosmosDBLexer(inputStream)
- stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
- p := parser.NewCosmosDBParser(stream)
-
- // Remove default error listener and add our own error listener.
- startPosition := &storepb.Position{Line: int32(baseLine) + 1}
- lexer.RemoveErrorListeners()
- lexerErrorListener := &base.ParseErrorListener{
- Statement: statement,
- StartPosition: startPosition,
- }
- lexer.AddErrorListener(lexerErrorListener)
-
- p.RemoveErrorListeners()
- parserErrorListener := &base.ParseErrorListener{
- Statement: statement,
- StartPosition: startPosition,
- }
- p.AddErrorListener(parserErrorListener)
-
- p.BuildParseTrees = true
-
- tree := p.Root()
-
- if lexerErrorListener.Err != nil {
- return nil, lexerErrorListener.Err
+// byteOffsetToRunePosition converts a byte offset to a 1-based line:column position
+// where column is measured in Unicode code points.
+func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {
+ if byteOffset > len(sql) {
+ byteOffset = len(sql)
}
- if parserErrorListener.Err != nil {
- return nil, parserErrorListener.Err
+ line := int32(1)
+ runeCol := int32(0)
+ i := 0
+ for i < byteOffset {
+ r, size := utf8.DecodeRuneInString(sql[i:])
+ if r == '\n' {
+ line++
+ runeCol = 0
+ } else {
+ runeCol++
+ }
+ i += size
}
- result := &base.ANTLRAST{
- StartPosition: startPosition,
- Tree: tree,
- Tokens: stream,
+ return &storepb.Position{
+ Line: line,
+ Column: runeCol + 1,
}
-
- return result, nil
}
diff --git a/backend/plugin/parser/cosmosdb/cosmosdb_test.go b/backend/plugin/parser/cosmosdb/cosmosdb_test.go
index 4c073d6c51..f45141e83a 100644
--- a/backend/plugin/parser/cosmosdb/cosmosdb_test.go
+++ b/backend/plugin/parser/cosmosdb/cosmosdb_test.go
@@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/require"
)
-func TestParseCosmosDBQuery(t *testing.T) {
+func TestParseCosmosDB(t *testing.T) {
tests := []struct {
name string
statement string
@@ -48,7 +48,7 @@ func TestParseCosmosDBQuery(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- results, err := ParseCosmosDBQuery(tt.statement)
+ results, err := ParseCosmosDB(tt.statement)
if tt.wantErr {
require.Error(t, err)
return
@@ -56,10 +56,9 @@ func TestParseCosmosDBQuery(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, tt.wantStatCount, len(results))
- // Verify each result has the required fields
for i, result := range results {
- assert.NotNil(t, result.Tree, "Result %d should have a Tree", i)
- assert.NotNil(t, result.Tokens, "Result %d should have Tokens", i)
+ assert.NotNil(t, result.Node, "Result %d should have a Node", i)
+ assert.NotEmpty(t, result.Text, "Result %d should have Text", i)
assert.NotNil(t, result.StartPosition, "Result %d should have StartPosition", i)
assert.Equal(t, int32(1), result.StartPosition.Line, "Result %d should have StartPosition.Line 1", i)
}
@@ -67,7 +66,7 @@ func TestParseCosmosDBQuery(t *testing.T) {
}
}
-func TestParseCosmosDBQueryErrors(t *testing.T) {
+func TestParseCosmosDBErrors(t *testing.T) {
tests := []struct {
name string
statement string
@@ -84,7 +83,7 @@ func TestParseCosmosDBQueryErrors(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- _, err := ParseCosmosDBQuery(tt.statement)
+ _, err := ParseCosmosDB(tt.statement)
require.Error(t, err, "Expected error for invalid CosmosDB SQL")
})
}
diff --git a/backend/plugin/parser/cosmosdb/query.go b/backend/plugin/parser/cosmosdb/query.go
index 30a0a16687..babe9f5842 100644
--- a/backend/plugin/parser/cosmosdb/query.go
+++ b/backend/plugin/parser/cosmosdb/query.go
@@ -10,7 +10,7 @@ func init() {
}
func validateQuery(statement string) (bool, bool, error) {
- _, err := ParseCosmosDBQuery(statement)
+ _, err := ParseCosmosDB(statement)
if err != nil {
return false, false, err
}
diff --git a/backend/plugin/parser/cosmosdb/query_span.go b/backend/plugin/parser/cosmosdb/query_span.go
index e54ab8ddc1..dbe3885af2 100644
--- a/backend/plugin/parser/cosmosdb/query_span.go
+++ b/backend/plugin/parser/cosmosdb/query_span.go
@@ -2,11 +2,13 @@ package cosmosdb
import (
"context"
+ "log/slog"
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/cosmosdb"
+ "github.com/bytebase/omni/cosmosdb/analysis"
+ "github.com/bytebase/omni/cosmosdb/ast"
"github.com/pkg/errors"
+ "github.com/bytebase/bytebase/backend/common/log"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -20,7 +22,7 @@ func GetQuerySpan(_ context.Context, _ base.GetQuerySpanContext, stmt base.State
}
func getQuerySpanImpl(statement string) (*base.QuerySpan, error) {
- parseResults, err := ParseCosmosDBQuery(statement)
+ parseResults, err := ParseCosmosDB(statement)
if err != nil {
return nil, err
}
@@ -36,153 +38,100 @@ func getQuerySpanImpl(statement string) (*base.QuerySpan, error) {
return nil, errors.Errorf("expecting only one statement to get query span, but got %d", len(parseResults))
}
- ast := parseResults[0].Tree
- querySpanResultListener := &querySpanResultListener{}
- antlr.ParseTreeWalkerDefault.Walk(querySpanResultListener, ast)
- if querySpanResultListener.err != nil {
- return nil, querySpanResultListener.err
+ selectStmt, ok := parseResults[0].Node.(*ast.SelectStmt)
+ if !ok {
+ return nil, errors.Errorf("expected SelectStmt, got %T", parseResults[0].Node)
}
- querySpanPredicatePathsListener := &querySpanPredicatePathsListener{}
- antlr.ParseTreeWalkerDefault.Walk(querySpanPredicatePathsListener, ast)
- if querySpanPredicatePathsListener.err != nil {
- return nil, querySpanPredicatePathsListener.err
- }
- return &base.QuerySpan{
- Results: querySpanResultListener.result,
- PredicatePaths: querySpanPredicatePathsListener.predicatePaths,
- }, nil
-}
-
-type querySpanResultListener struct {
- *parser.BaseCosmosDBParserListener
-
- result []base.QuerySpanResult
-
- err error
-}
-
-func (l *querySpanResultListener) EnterSelect(ctx *parser.SelectContext) {
- if ctx.Select_clause().Select_specification().MULTIPLY_OPERATOR() != nil || ctx.From_clause() == nil {
- l.result = []base.QuerySpanResult{
- {
- Name: "",
- SourceFieldPaths: make(map[string][]*base.PathAST),
- SelectAsterisk: true,
- },
- }
- return
- }
+ qa := analysis.Analyze(selectStmt)
- containerName, fromAlias := extractFromNames(ctx.From_clause())
+ // Convert projections.
sourceFieldPaths := make(map[string][]*base.PathAST)
- for _, property := range ctx.Select_clause().Select_specification().Object_property_list().AllObject_property() {
- paths, name := extractPathsFromObjectProperty(property, containerName, fromAlias)
- // For VALUE expressions without an alias (e.g., SELECT VALUE CONCAT(c.name, c.email) FROM c),
- // the name is empty but we still store paths so scalar masking can detect sensitive fields.
- for _, path := range paths {
- if len(path) == 0 {
- continue
+ if !qa.SelectStar {
+ for _, proj := range qa.Projections {
+ for _, fp := range proj.SourcePaths {
+ if len(fp) == 0 {
+ continue
+ }
+ sourceFieldPaths[proj.Name] = append(sourceFieldPaths[proj.Name], fieldPathToPathAST(fp))
}
- sourceFieldPaths[name] = append(sourceFieldPaths[name], buildPathAST(path))
}
}
- l.result = []base.QuerySpanResult{
- {
- Name: "",
- SourceFieldPaths: sourceFieldPaths,
- SelectAsterisk: false,
- },
- }
-}
-
-// extractPathsFromObjectProperty extracts all source field paths and the output name
-// from a SELECT object_property.
-func extractPathsFromObjectProperty(ctx parser.IObject_propertyContext, containerName, fromAlias string) ([][]base.SelectorNode, string) {
- if ctx == nil {
- return nil, ""
- }
- paths := extractAllFieldPaths(ctx.Scalar_expression(), containerName, fromAlias)
- var propertyName string
- if ctx.Property_alias() != nil {
- propertyName = ctx.Property_alias().Identifier().GetText()
- }
- if propertyName == "" && len(paths) == 1 && len(paths[0]) > 0 {
- propertyName = paths[0][len(paths[0])-1].GetIdentifier()
- }
- return paths, propertyName
-}
-// extractAllFieldPaths recursively collects ALL field paths referenced in a scalar expression.
-func extractAllFieldPaths(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- if ctx == nil {
- return nil
+ result := base.QuerySpanResult{
+ SourceFieldPaths: sourceFieldPaths,
+ SelectAsterisk: qa.SelectStar,
}
- // Input alias (leaf).
- if ctx.Input_alias() != nil {
- return resolveInputAlias(ctx, containerName, fromAlias)
- }
- // Property access: a.b
- if ctx.DOT_SYMBOL() != nil {
- return extractFieldPathsDot(ctx, containerName, fromAlias)
- }
- // Bracket access: a["b"] or a[0]
- if ctx.LS_BRACKET_SYMBOL() != nil && ctx.IN_SYMBOL() == nil {
- return extractFieldPathsBracket(ctx, containerName, fromAlias)
- }
- // Unary / NOT
- if ctx.Unary_operator() != nil || (ctx.NOT_SYMBOL() != nil && len(ctx.AllScalar_expression()) == 1) {
- return extractAllFieldPaths(ctx.Scalar_expression(0), containerName, fromAlias)
- }
- // Function calls
- if ctx.Scalar_function_expression() != nil {
- return extractFieldPathsFromFunction(ctx.Scalar_function_expression(), containerName, fromAlias)
- }
- // Binary/multi-child expressions: collect from all children.
- if allExprs := ctx.AllScalar_expression(); len(allExprs) >= 2 {
- return collectFieldPathsFromAll(allExprs, containerName, fromAlias)
+ // Convert predicates.
+ var predicatePaths map[string]*base.PathAST
+ if len(qa.Predicates) > 0 {
+ predicatePaths = make(map[string]*base.PathAST)
+ for _, fp := range qa.Predicates {
+ if len(fp) == 0 {
+ continue
+ }
+ pathAST := fieldPathToPathAST(fp)
+ str, err := pathAST.String()
+ if err != nil {
+ slog.Warn("failed to convert path ast to string", log.BBError(err))
+ continue
+ }
+ predicatePaths[str] = pathAST
+ }
}
- return nil
-}
-func extractFieldPathsDot(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- paths := extractAllFieldPaths(ctx.Scalar_expression(0), containerName, fromAlias)
- propName := ctx.Property_name().Identifier().GetText()
- for i := range paths {
- paths[i] = append(paths[i], base.NewItemSelector(propName))
- }
- return paths
+ return &base.QuerySpan{
+ Results: []base.QuerySpanResult{result},
+ PredicatePaths: predicatePaths,
+ }, nil
}
-func extractFieldPathsBracket(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- paths := extractAllFieldPaths(ctx.Scalar_expression(0), containerName, fromAlias)
- for i := range paths {
- appendBracketSelector(ctx, paths, i)
+// fieldPathToPathAST converts an omni analysis.FieldPath to a base.PathAST.
+//
+// The omni analysis represents array access as a separate Selector with
+// Name=indexText and ArrayIndex>=0, following the preceding property selector.
+// The base.PathAST representation merges these: an ArraySelector carries the
+// property name AND the index. So we fold consecutive (ItemSelector, ArraySelector)
+// pairs into a single base.ArraySelector.
+func fieldPathToPathAST(fp analysis.FieldPath) *base.PathAST {
+ if len(fp) == 0 {
+ return nil
}
- return paths
-}
-func extractFieldPathsFromFunction(ctx parser.IScalar_function_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- if ctx == nil {
+ // Merge omni selectors into base nodes, folding array indices into the
+ // preceding item selector.
+ nodes := mergeSelectors(fp)
+ if len(nodes) == 0 {
return nil
}
- var exprs []parser.IScalar_expressionContext
- switch {
- case ctx.Udf_scalar_function_expression() != nil:
- exprs = ctx.Udf_scalar_function_expression().AllScalar_expression()
- case ctx.Builtin_function_expression() != nil:
- exprs = ctx.Builtin_function_expression().AllScalar_expression()
- default:
- return nil
+
+ pathAST := base.NewPathAST(nodes[0])
+ current := pathAST.Root
+ for i := 1; i < len(nodes); i++ {
+ current.SetNext(nodes[i])
+ current = nodes[i]
}
- return collectFieldPathsFromAll(exprs, containerName, fromAlias)
+ return pathAST
}
-func collectFieldPathsFromAll(exprs []parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- var all [][]base.SelectorNode
- for _, expr := range exprs {
- all = append(all, extractAllFieldPaths(expr, containerName, fromAlias)...)
+// mergeSelectors converts omni selectors to base.SelectorNode values,
+// merging an array selector into the preceding item selector when possible.
+func mergeSelectors(fp analysis.FieldPath) []base.SelectorNode {
+ var nodes []base.SelectorNode
+ for i := 0; i < len(fp); i++ {
+ s := fp[i]
+ if s.IsArray() && len(nodes) > 0 {
+ // Merge with the preceding item selector: replace it with an ArraySelector
+ // that keeps the property name but adds the array index.
+ prev := nodes[len(nodes)-1]
+ nodes[len(nodes)-1] = base.NewArraySelector(prev.GetIdentifier(), s.ArrayIndex)
+ } else if s.IsArray() {
+ // Array selector with no preceding item -- keep as-is (shouldn't normally happen).
+ nodes = append(nodes, base.NewArraySelector(s.Name, s.ArrayIndex))
+ } else {
+ nodes = append(nodes, base.NewItemSelector(s.Name))
+ }
}
- return all
+ return nodes
}
diff --git a/backend/plugin/parser/cosmosdb/query_span_helpers.go b/backend/plugin/parser/cosmosdb/query_span_helpers.go
deleted file mode 100644
index 439b28c6c3..0000000000
--- a/backend/plugin/parser/cosmosdb/query_span_helpers.go
+++ /dev/null
@@ -1,76 +0,0 @@
-package cosmosdb
-
-import (
- "log/slog"
- "strconv"
-
- parser "github.com/bytebase/parser/cosmosdb"
-
- "github.com/bytebase/bytebase/backend/plugin/parser/base"
-)
-
-// extractFromNames returns the container name and from-clause alias.
-func extractFromNames(fromClause parser.IFrom_clauseContext) (containerName, fromAlias string) {
- fromSource := fromClause.From_specification().From_source()
- containerExpr := fromSource.Container_expression()
- if containerExpr != nil && containerExpr.Container_name() != nil {
- if i := containerExpr.Container_name().Identifier(); i != nil {
- containerName = i.GetText()
- }
- }
- if i := fromSource.Identifier(); i != nil {
- fromAlias = i.GetText()
- }
- return containerName, fromAlias
-}
-
-// resolveInputAlias returns a single-element path for an input alias, resolving from-clause alias.
-func resolveInputAlias(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- name := ctx.Input_alias().Identifier().GetText()
- if fromAlias != "" && name == fromAlias {
- name = containerName
- }
- return [][]base.SelectorNode{{base.NewItemSelector(name)}}
-}
-
-// buildPathAST converts a slice of SelectorNodes into a linked PathAST.
-func buildPathAST(path []base.SelectorNode) *base.PathAST {
- ast := base.NewPathAST(path[0])
- next := ast.Root
- for i := 1; i < len(path); i++ {
- next.SetNext(path[i])
- next = next.GetNext()
- }
- return ast
-}
-
-// unquote strips the first and last character from a quoted string literal.
-func unquote(s string) string {
- if len(s) > 1 {
- return s[1 : len(s)-1]
- }
- return s
-}
-
-// appendBracketSelector appends the bracket-access selector (string key or array index)
-// to the path at index i. Shared by query_span.go and query_span_predicate.go.
-func appendBracketSelector(ctx parser.IScalar_expressionContext, paths [][]base.SelectorNode, i int) {
- switch {
- case ctx.DOUBLE_QUOTE_STRING_LITERAL() != nil:
- paths[i] = append(paths[i], base.NewItemSelector(unquote(ctx.DOUBLE_QUOTE_STRING_LITERAL().GetText())))
- case ctx.SINGLE_QUOTE_STRING_LITERAL() != nil:
- paths[i] = append(paths[i], base.NewItemSelector(unquote(ctx.SINGLE_QUOTE_STRING_LITERAL().GetText())))
- case ctx.Array_index() != nil:
- if len(paths[i]) == 0 {
- return
- }
- index, err := strconv.Atoi(ctx.Array_index().GetText())
- if err != nil {
- slog.Warn("cannot convert array index to int", slog.String("index", ctx.Array_index().GetText()))
- return
- }
- last := paths[i][len(paths[i])-1]
- paths[i][len(paths[i])-1] = base.NewArraySelector(last.GetIdentifier(), index)
- default:
- }
-}
diff --git a/backend/plugin/parser/cosmosdb/query_span_predicate.go b/backend/plugin/parser/cosmosdb/query_span_predicate.go
deleted file mode 100644
index f969e36063..0000000000
--- a/backend/plugin/parser/cosmosdb/query_span_predicate.go
+++ /dev/null
@@ -1,147 +0,0 @@
-package cosmosdb
-
-import (
- "log/slog"
-
- parser "github.com/bytebase/parser/cosmosdb"
-
- "github.com/bytebase/bytebase/backend/common/log"
- "github.com/bytebase/bytebase/backend/plugin/parser/base"
-)
-
-type querySpanPredicatePathsListener struct {
- *parser.BaseCosmosDBParserListener
-
- predicatePaths map[string]*base.PathAST
- err error
-}
-
-func (l *querySpanPredicatePathsListener) EnterSelect(ctx *parser.SelectContext) {
- whereClause := ctx.Where_clause()
- if whereClause == nil {
- return
- }
- fromClause := ctx.From_clause()
- if fromClause == nil {
- return
- }
-
- originalContainerName, fromIdentifier := extractFromNames(fromClause)
- l.predicatePaths = buildPredicatePaths(whereClause.Scalar_expression(), originalContainerName, fromIdentifier)
-}
-
-func buildPredicatePaths(expr parser.IScalar_expressionContext, containerName, fromAlias string) map[string]*base.PathAST {
- paths := extractPredicateFields(expr, containerName, fromAlias)
- r := make(map[string]*base.PathAST)
- for _, path := range paths {
- if len(path) == 0 {
- continue
- }
- ast := buildPathAST(path)
- str, err := ast.String()
- if err != nil {
- slog.Warn("failed to convert path ast to string", log.BBError(err))
- }
- r[str] = ast
- }
- return r
-}
-
-// extractPredicateFields collects all field paths referenced in a scalar expression used as a predicate.
-func extractPredicateFields(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- if ctx == nil {
- return nil
- }
-
- // Input alias (leaf field reference).
- if ctx.Input_alias() != nil {
- return resolveInputAlias(ctx, containerName, fromAlias)
- }
- // Property access: a.b.c
- if ctx.DOT_SYMBOL() != nil {
- return extractPredicateDotAccess(ctx, containerName, fromAlias)
- }
- // Bracket access: a["b"] or a[0] (but not IN expression which also uses brackets).
- if ctx.LS_BRACKET_SYMBOL() != nil && ctx.IN_SYMBOL() == nil {
- return extractPredicateBracketAccess(ctx, containerName, fromAlias)
- }
- // Unary operator or standalone NOT.
- if ctx.Unary_operator() != nil || (ctx.NOT_SYMBOL() != nil && len(ctx.AllScalar_expression()) == 1) {
- return extractPredicateFields(ctx.Scalar_expression(0), containerName, fromAlias)
- }
- // Parenthesized expression (not subquery, not EXISTS).
- if ctx.LR_BRACKET_SYMBOL() != nil && ctx.Select_() == nil && ctx.EXISTS_SYMBOL() == nil {
- return extractPredicateFields(ctx.Scalar_expression(0), containerName, fromAlias)
- }
- // Function calls.
- if ctx.Scalar_function_expression() != nil {
- return extractPredicateFieldsFromFunction(ctx.Scalar_function_expression(), containerName, fromAlias)
- }
- // Ternary operator: cond ? a : b
- if ctx.QUESTION_MARK_SYMBOL() != nil {
- return collectPredicateFieldsFromAll(ctx.AllScalar_expression(), containerName, fromAlias)
- }
- // Binary operators (comparison, arithmetic, bitwise, logical, IN, BETWEEN, LIKE, concat).
- // All share the same logic: collect fields from all child expressions.
- if isBinaryOrMultiChildExpression(ctx) {
- return collectPredicateFieldsFromAll(ctx.AllScalar_expression(), containerName, fromAlias)
- }
- return nil
-}
-
-func extractPredicateDotAccess(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- paths := extractPredicateFields(ctx.Scalar_expression(0), containerName, fromAlias)
- propName := ctx.Property_name().Identifier().GetText()
- for i := range paths {
- paths[i] = append(paths[i], base.NewItemSelector(propName))
- }
- return paths
-}
-
-func extractPredicateBracketAccess(ctx parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- paths := extractPredicateFields(ctx.Scalar_expression(0), containerName, fromAlias)
- for i := range paths {
- appendBracketSelector(ctx, paths, i)
- }
- return paths
-}
-
-func extractPredicateFieldsFromFunction(ctx parser.IScalar_function_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- var exprs []parser.IScalar_expressionContext
- switch {
- case ctx.Udf_scalar_function_expression() != nil:
- exprs = ctx.Udf_scalar_function_expression().AllScalar_expression()
- case ctx.Builtin_function_expression() != nil:
- exprs = ctx.Builtin_function_expression().AllScalar_expression()
- default:
- return nil
- }
- return collectPredicateFieldsFromAll(exprs, containerName, fromAlias)
-}
-
-// isBinaryOrMultiChildExpression returns true for binary operators and multi-child expressions
-// that all share the same extraction logic: collect fields from all child scalar expressions.
-func isBinaryOrMultiChildExpression(ctx parser.IScalar_expressionContext) bool {
- return ctx.Comparison_operator() != nil ||
- ctx.Multiplicative_operator() != nil ||
- ctx.Additive_operator() != nil ||
- ctx.Shift_operator() != nil ||
- ctx.BIT_AND_SYMBOL() != nil ||
- ctx.BIT_XOR_SYMBOL() != nil ||
- ctx.BIT_OR_SYMBOL() != nil ||
- ctx.DOUBLE_BAR_SYMBOL() != nil ||
- ctx.LIKE_SYMBOL() != nil ||
- ctx.IN_SYMBOL() != nil ||
- ctx.BETWEEN_SYMBOL() != nil ||
- // AND/OR logical operators. BETWEEN also sets AND_SYMBOL, but BETWEEN is checked above.
- ctx.AND_SYMBOL() != nil ||
- ctx.OR_SYMBOL() != nil
-}
-
-func collectPredicateFieldsFromAll(exprs []parser.IScalar_expressionContext, containerName, fromAlias string) [][]base.SelectorNode {
- var all [][]base.SelectorNode
- for _, expr := range exprs {
- all = append(all, extractPredicateFields(expr, containerName, fromAlias)...)
- }
- return all
-}
diff --git a/backend/plugin/parser/cosmosdb/query_span_test.go b/backend/plugin/parser/cosmosdb/query_span_test.go
index 6f8981876b..c357203d61 100644
--- a/backend/plugin/parser/cosmosdb/query_span_test.go
+++ b/backend/plugin/parser/cosmosdb/query_span_test.go
@@ -20,7 +20,7 @@ func TestGetQuerySpan(t *testing.T) {
}
var (
- record = true
+ record = false
testDataPaths = []string{
"test-data/query-span/standard.yaml",
}
@@ -28,8 +28,6 @@ func TestGetQuerySpan(t *testing.T) {
a := require.New(t)
for _, testDataPath := range testDataPaths {
- testDataPath := testDataPath
-
yamlFile, err := os.Open(testDataPath)
a.NoError(err)
diff --git a/backend/plugin/parser/cosmosdb/split.go b/backend/plugin/parser/cosmosdb/split.go
index 1a7bf2d7aa..4a2781f7c0 100644
--- a/backend/plugin/parser/cosmosdb/split.go
+++ b/backend/plugin/parser/cosmosdb/split.go
@@ -3,10 +3,8 @@ package cosmosdb
import (
"strings"
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/cosmosdb"
+ omnicosmosdb "github.com/bytebase/omni/cosmosdb"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -15,70 +13,36 @@ func init() {
base.RegisterSplitterFunc(storepb.Engine_COSMOSDB, SplitSQL)
}
-// SplitSQL splits the given SQL statement into multiple SQL statements.
-// Note: CosmosDB only supports single SELECT statements, so this returns the entire input as one statement.
+// SplitSQL splits the given SQL statement into individual statements.
+// CosmosDB only supports single SELECT statements.
func SplitSQL(statement string) ([]base.Statement, error) {
- // CosmosDB doesn't support multiple statements or semicolon delimiters.
- // The grammar only accepts: root: select EOF
if strings.TrimSpace(statement) == "" {
return nil, nil
}
- // Use lexer to get proper position information
- lexer := parser.NewCosmosDBLexer(antlr.NewInputStream(statement))
- stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
- stream.Fill()
-
- tokens := stream.GetAllTokens()
- if len(tokens) == 0 {
- return nil, nil
+ stmts, err := omnicosmosdb.Parse(statement)
+ if err != nil {
+ return nil, err
}
-
- // Check if statement is empty (only whitespace/EOF)
- empty := true
- for _, token := range tokens {
- if token.GetTokenType() == antlr.TokenEOF {
- continue
- }
- if token.GetChannel() != antlr.TokenHiddenChannel {
- empty = false
- break
- }
- }
-
- var lastToken antlr.Token
- for _, token := range tokens {
- if token.GetTokenType() == antlr.TokenEOF {
- break
- }
- if token.GetChannel() == antlr.TokenDefaultChannel {
- lastToken = token
- }
+ if len(stmts) == 0 {
+ return nil, nil
}
- if lastToken == nil && len(tokens) > 0 {
- lastToken = tokens[len(tokens)-1]
- }
+ var result []base.Statement
+ for _, stmt := range stmts {
+ startPos := byteOffsetToRunePosition(statement, stmt.ByteStart)
+ endPos := byteOffsetToRunePosition(statement, stmt.ByteEnd)
- // Since Text is the entire statement (including any leading whitespace),
- // Start should point to the first character (position 1,1)
- return []base.Statement{
- {
- Text: statement,
+ result = append(result, base.Statement{
+ Text: stmt.Text,
Range: &storepb.Range{
- Start: 0,
- End: int32(len(statement)),
+ Start: int32(stmt.ByteStart),
+ End: int32(stmt.ByteEnd),
},
- Start: &storepb.Position{
- Line: 1,
- Column: 1,
- },
- End: common.ConvertANTLRTokenToExclusiveEndPosition(
- int32(lastToken.GetLine()),
- int32(lastToken.GetColumn()),
- lastToken.GetText(),
- ),
- Empty: empty,
- },
- }, nil
+ Start: startPos,
+ End: endPos,
+ Empty: stmt.Empty(),
+ })
+ }
+ return result, nil
}
diff --git a/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml b/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml
index 8d9338dd37..0a94ac64a4 100644
--- a/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml
+++ b/backend/plugin/parser/cosmosdb/test-data/query-span/standard.yaml
@@ -14,7 +14,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT TOP with star
statement: SELECT TOP 10 * FROM c
querySpan:
@@ -27,7 +26,6 @@
selectasterisk: true
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT TOP with field projection
statement: SELECT TOP 5 c.name, c.email FROM container c
querySpan:
@@ -44,7 +42,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT with ORDER BY
statement: SELECT c.name, c.population FROM container c ORDER BY c.population DESC
querySpan:
@@ -61,7 +58,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT with OFFSET LIMIT
statement: SELECT c.name FROM container c ORDER BY c.name OFFSET 0 LIMIT 10
querySpan:
@@ -76,7 +72,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT with function in projection
statement: SELECT UPPER(c.name) AS upperName, c.email FROM container c
querySpan:
@@ -93,7 +88,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT with aggregation
statement: SELECT COUNT(1) AS totalRecords FROM c
querySpan:
@@ -106,7 +100,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT with GROUP BY
statement: SELECT c.country, COUNT(1) AS cnt FROM container c GROUP BY c.country
querySpan:
@@ -121,7 +114,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT DISTINCT VALUE
statement: SELECT DISTINCT VALUE c.country FROM container c
querySpan:
@@ -136,7 +128,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT VALUE
statement: SELECT VALUE c.name FROM container c
querySpan:
@@ -151,7 +142,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: SELECT with underscore fields
statement: SELECT c._ts, c._etag, c.name FROM container c
querySpan:
@@ -170,7 +160,6 @@
selectasterisk: false
sourcecolumns: []
predicatecolumns: []
- predicatepaths: []
- description: WHERE with not-equal operator
statement: SELECT * FROM container c WHERE c.status != "inactive"
querySpan:
diff --git a/backend/plugin/parser/cosmosdb/test-data/test_split.yaml b/backend/plugin/parser/cosmosdb/test-data/test_split.yaml
index 50287e2a78..7117931704 100644
--- a/backend/plugin/parser/cosmosdb/test-data/test_split.yaml
+++ b/backend/plugin/parser/cosmosdb/test-data/test_split.yaml
@@ -50,30 +50,30 @@
- description: 'position semantic: leading newlines'
input: "\n\nSELECT 1"
result:
- - text: "\n\nSELECT 1"
- baseline: 0
+ - text: "SELECT 1"
+ baseline: 2
start:
- line: 1
+ line: 3
column: 1
end:
line: 3
column: 9
range:
- start: 0
+ start: 2
end: 10
empty: false
- description: 'position semantic: leading spaces and newlines'
input: "\n \n SELECT 1"
result:
- - text: "\n \n SELECT 1"
- baseline: 0
+ - text: "SELECT 1"
+ baseline: 2
start:
- line: 1
- column: 1
+ line: 3
+ column: 3
end:
line: 3
column: 11
range:
- start: 0
+ start: 6
end: 14
empty: false
diff --git a/go.mod b/go.mod
index ffef2c3974..97b59c3e7a 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
github.com/bmatcuk/doublestar/v4 v4.10.0
github.com/bytebase/gomongo v0.0.0-20260324061529-f15b9a5add35
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0
- github.com/bytebase/omni v0.0.0-20260330073149-2f02f119f9ec
+ github.com/bytebase/omni v0.0.0-20260331094502-07f157a265aa
github.com/caarlos0/env/v11 v11.4.0
github.com/cenkalti/backoff/v5 v5.0.3
github.com/cockroachdb/cockroachdb-parser v0.25.2
diff --git a/go.sum b/go.sum
index 0f7c7055df..4bf03161d2 100644
--- a/go.sum
+++ b/go.sum
@@ -253,8 +253,8 @@ github.com/bytebase/gomongo v0.0.0-20260324061529-f15b9a5add35 h1:SjcZSrUMNj6GZq
github.com/bytebase/gomongo v0.0.0-20260324061529-f15b9a5add35/go.mod h1:jHdAD+tN3bO2Hazme0crgkNiHqkDR/3Igesy1chhjLA=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=
-github.com/bytebase/omni v0.0.0-20260330073149-2f02f119f9ec h1:cgkOoqoxOaTM1BXjzp0359Idbboltho8bmYMZunQ2OM=
-github.com/bytebase/omni v0.0.0-20260330073149-2f02f119f9ec/go.mod h1:1HI7B1k/5jH/Po89iGM6fMoz+Jh1PqUHfgeudpZSdkk=
+github.com/bytebase/omni v0.0.0-20260331094502-07f157a265aa h1:o1a+V1HLaXt/Rv2atFZaEQv2+YoZrimKHvC0BUgKN6I=
+github.com/bytebase/omni v0.0.0-20260331094502-07f157a265aa/go.mod h1:7kUVQrivlkckALafCX5j4W3M95Cg9+Kw69ZukIHKXhQ=
github.com/bytebase/parser v0.0.0-20260324092042-1d52e5412f50 h1:Kq5wlhZ586Rch39y/IQV9MnrEzHob4iiXBG6zrbvSMU=
github.com/bytebase/parser v0.0.0-20260324092042-1d52e5412f50/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=
github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=