Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(mssql): migrate resource_change extractor from ANTLR to omni (#20059)
rebelice5af712652456
diff --git a/backend/plugin/db/mssql/resource_change_integration_testcontainer_test.go b/backend/plugin/db/mssql/resource_change_integration_testcontainer_test.gonew file mode 100644index 0000000000..8c2659e531--- /dev/null+++ b/backend/plugin/db/mssql/resource_change_integration_testcontainer_test.go@@ -0,0 +1,180 @@+package mssql++import (+ "context"+ "fmt"+ "strconv"+ "strings"+ "testing"+ "time"++ _ "github.com/microsoft/go-mssqldb"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"+ _ "github.com/bytebase/bytebase/backend/plugin/parser/tsql" // register MSSQL statement parser+)++// TestResourceChangeIntegrationMSSQL verifies the resource_change.go migration+// against a real SQL Server. It targets three claims made during review:+// 1. INSERT ... DEFAULT VALUES: our new code skips EXPLAIN and directly+// counts 1 row. Proving SHOWPLAN would also return 1 confirms the final+// affected-rows number is unchanged.+// 2. A statement whose text carries a leading SQL comment (from the+// split-layer's gap-absorption behavior) must still be accepted by+// SHOWPLAN_ALL so the downstream estimate is unaffected.+// 3. DROP INDEX ... ON t registers t in ChangedResources (omni 51d40fb+// exposed the target table; migration now consumes it).+//+//nolint:tparallel+func TestResourceChangeIntegrationMSSQL(t *testing.T) {+ ctx := context.Background()+ container := testcontainer.GetTestMSSQLContainer(ctx, t)+ defer container.Close(ctx)++ host := container.GetHost()+ portInt, err := strconv.Atoi(container.GetPort())+ require.NoError(t, err)++ dbName := fmt.Sprintf("resource_change_%d", time.Now().UnixNano())+ driver := openMSSQL(ctx, t, host, portInt, "master")+ _, err = driver.Execute(ctx, fmt.Sprintf("CREATE DATABASE [%s]", dbName), db.ExecuteOptions{CreateDatabase: true})+ require.NoError(t, err)+ driver.Close(ctx)++ defer func() {+ cleanup := openMSSQL(ctx, t, host, portInt, "master")+ defer cleanup.Close(ctx)+ _, _ = cleanup.Execute(ctx, fmt.Sprintf("ALTER DATABASE [%s] SET SINGLE_USER WITH ROLLBACK IMMEDIATE", dbName), db.ExecuteOptions{CreateDatabase: true})+ _, _ = cleanup.Execute(ctx, fmt.Sprintf("DROP DATABASE [%s]", dbName), db.ExecuteOptions{CreateDatabase: true})+ }()++ driver = openMSSQL(ctx, t, host, portInt, dbName)+ defer driver.Close(ctx)+ mssqlDriver, ok := driver.(*Driver)+ require.True(t, ok, "expected *Driver")++ // Seed a small table with 5 rows so UPDATE/DELETE have something to plan against.+ setup := []string{+ `CREATE TABLE dbo.t (id INT IDENTITY(1,1) PRIMARY KEY, c1 INT NULL)`,+ `INSERT INTO dbo.t (c1) VALUES (1),(2),(3),(4),(5)`,+ `CREATE INDEX idx_c1 ON dbo.t(c1)`,+ }+ for _, s := range setup {+ _, err := driver.Execute(ctx, s, db.ExecuteOptions{})+ require.NoError(t, err, "setup: %s", s)+ }++ t.Run("INSERT_DEFAULT_VALUES_parity", func(t *testing.T) {+ sql := `INSERT INTO dbo.t DEFAULT VALUES;`++ // Path 1 (new code): extractor must route this into InsertCount with no sample.+ summary := summarize(t, sql)+ require.Equal(t, 1, summary.InsertCount, "DEFAULT VALUES should be counted as 1 known insert")+ require.Equal(t, 0, summary.DMLCount, "DEFAULT VALUES must not be sent to EXPLAIN")+ require.Empty(t, summary.SampleDMLS, "no sample text should be emitted for DEFAULT VALUES")++ // Path 2 (proof of parity): if we HAD sent it through SHOWPLAN, what would it return?+ // Matches the old ANTLR code's dmlCount+sample path. Must be 1 so the final affected-rows+ // number is identical between old and new.+ showplanRows, err := mssqlDriver.CountAffectedRows(ctx, sql)+ require.NoError(t, err)+ require.EqualValues(t, 1, showplanRows, "SHOWPLAN_ALL must also estimate DEFAULT VALUES as 1 row")+ })++ t.Run("UPDATE_with_leading_comment_showplan_ok", func(t *testing.T) {+ // New code's SampleDMLS may contain a leading comment because+ // split.go absorbs inter-statement gaps into the next statement.+ // Assert that SQL Server's SHOWPLAN_ALL accepts such text verbatim+ // and returns a usable EstimateRows, so the downstream number is+ // not affected by the cosmetic regression.+ sampleAsSentToShowplan := "-- header comment\nUPDATE dbo.t SET c1 = 99 WHERE id = 1;"+ rows, err := mssqlDriver.CountAffectedRows(ctx, sampleAsSentToShowplan)+ require.NoError(t, err, "SHOWPLAN should accept leading comments")+ require.EqualValues(t, 1, rows, "UPDATE of id=1 should estimate 1 row regardless of comment prefix")++ // Control: same UPDATE with no comment prefix should yield the same estimate.+ rows2, err := mssqlDriver.CountAffectedRows(ctx, "UPDATE dbo.t SET c1 = 99 WHERE id = 1;")+ require.NoError(t, err)+ require.Equal(t, rows, rows2, "leading comment must not change the estimate")+ })++ t.Run("INSERT_VALUES_multi_row_insertCount", func(t *testing.T) {+ sql := `INSERT INTO dbo.t (c1) VALUES (10),(20),(30);`+ summary := summarize(t, sql)+ require.Equal(t, 3, summary.InsertCount)+ require.Equal(t, 0, summary.DMLCount)+ require.Empty(t, summary.SampleDMLS)+ })++ t.Run("DROP_INDEX_tracks_target_table", func(t *testing.T) {+ sql := `DROP INDEX idx_c1 ON dbo.t;`+ summary := summarize(t, sql)++ proto := summary.ChangedResources.Build()+ found := false+ for _, database := range proto.Databases {+ for _, schema := range database.Schemas {+ for _, tbl := range schema.Tables {+ if schema.Name == "dbo" && tbl.Name == "t" {+ found = true+ }+ }+ }+ }+ require.True(t, found, "DROP INDEX ... ON dbo.t should register dbo.t as a changed table (omni 51d40fb)")++ // Sanity: executing the DROP INDEX against the seeded DB should succeed,+ // proving the SQL we parsed is actually runnable.+ _, err := driver.Execute(ctx, sql, db.ExecuteOptions{})+ require.NoError(t, err)+ })++ t.Run("UPDATE_goes_through_explain", func(t *testing.T) {+ sql := `UPDATE dbo.t SET c1 = c1 + 1 WHERE id > 2;`+ summary := summarize(t, sql)+ require.Equal(t, 0, summary.InsertCount)+ require.Equal(t, 1, summary.DMLCount)+ require.Len(t, summary.SampleDMLS, 1)+ require.Equal(t, strings.TrimSpace(sql), summary.SampleDMLS[0])++ rows, err := mssqlDriver.CountAffectedRows(ctx, summary.SampleDMLS[0])+ require.NoError(t, err)+ // Seeded 5 rows with id 1..5; predicate matches ids 3,4,5 → 3 rows.+ require.EqualValues(t, 3, rows)+ })+}++func summarize(t *testing.T, sql string) *parserbase.ChangeSummary {+ t.Helper()+ stmts, err := parserbase.ParseStatements(storepb.Engine_MSSQL, sql)+ require.NoError(t, err)+ asts := parserbase.ExtractASTs(stmts)+ summary, err := parserbase.ExtractChangedResources(storepb.Engine_MSSQL, "resource_change_test", "dbo", nil, asts, sql)+ require.NoError(t, err)+ return summary+}++func openMSSQL(ctx context.Context, t *testing.T, host string, port int, database string) db.Driver {+ t.Helper()+ driverInstance := &Driver{}+ cfg := db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Type: storepb.DataSourceType_ADMIN,+ Username: "sa",+ Host: host,+ Port: strconv.Itoa(port),+ Database: database,+ },+ Password: "Test123!",+ ConnectionContext: db.ConnectionContext{+ DatabaseName: database,+ },+ }+ driver, err := driverInstance.Open(ctx, storepb.Engine_MSSQL, cfg)+ require.NoError(t, err)+ return driver+}diff --git a/backend/plugin/parser/tsql/resource_change.go b/backend/plugin/parser/tsql/resource_change.goindex 8747e8b111..c46b05be5f 100644--- a/backend/plugin/parser/tsql/resource_change.go+++ b/backend/plugin/parser/tsql/resource_change.go@@ -4,8 +4,7 @@ import ("strings""unicode"- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/tsql"+ "github.com/bytebase/omni/mssql/ast""github.com/pkg/errors""github.com/bytebase/bytebase/backend/common"@@ -19,244 +18,122 @@ func init() {base.RegisterExtractChangedResourcesFunc(storepb.Engine_MSSQL, extractChangedResources)}-func extractChangedResources(currentDatabase string, currentSchema string, dbMetadata *model.DatabaseMetadata, asts []base.AST, statement string) (*base.ChangeSummary, error) {+func extractChangedResources(currentDatabase string, currentSchema string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {changedResources := model.NewChangedResources(dbMetadata)- l := &tsqlChangedResourceExtractListener{- currentDatabase: currentDatabase,- currentSchema: currentSchema,- dbMetadata: dbMetadata,- changedResources: changedResources,- statement: statement,- }-- for _, ast := range asts {- antlrAST, ok := base.GetANTLRAST(ast)- if !ok {- return nil, errors.New("expected ANTLR AST for MSSQL")- }- antlr.ParseTreeWalkerDefault.Walk(l, antlrAST.Tree)- }-- return &base.ChangeSummary{- ChangedResources: changedResources,- SampleDMLS: l.sampleDMLs,- DMLCount: l.dmlCount,- InsertCount: l.insertCount,- }, nil-}--type tsqlChangedResourceExtractListener struct {- *parser.BaseTSqlParserListener-- currentDatabase string- currentSchema string- dbMetadata *model.DatabaseMetadata- changedResources *model.ChangedResources- statement string- sampleDMLs []string- dmlCount int- insertCount int-- // Internal data structure used temporarily.- text string-}--func (l *tsqlChangedResourceExtractListener) EnterSql_clauses(ctx *parser.Sql_clausesContext) {- l.text = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)-}--func (l *tsqlChangedResourceExtractListener) EnterCreate_table(ctx *parser.Create_tableContext) {- tableName := ctx.Table_name()- if tableName == nil {- return- }- d, s, t := normalizeTableNameSeparated(tableName, l.currentDatabase, l.currentSchema, false /* caseSensitive */)-- l.changedResources.AddTable(- d,- s,- &storepb.ChangedResourceTable{- Name: t,- },- false)-}--func (l *tsqlChangedResourceExtractListener) EnterDrop_table(ctx *parser.Drop_tableContext) {- if ctx.AllTable_name() == nil {- return- }- for _, tableName := range ctx.AllTable_name() {- d, s, t := normalizeTableNameSeparated(tableName, l.currentDatabase, l.currentSchema, false /* caseSensitive */)+ var dmlCount, insertCount int+ var sampleDMLs []string- l.changedResources.AddTable(- d,- s,- &storepb.ChangedResourceTable{- Name: t,- },- true)- }-}--func (l *tsqlChangedResourceExtractListener) EnterAlter_table(ctx *parser.Alter_tableContext) {- if ctx.AllTable_name() == nil {- return- }-- for _, tableName := range ctx.AllTable_name() {- d, s, t := normalizeTableNameSeparated(tableName, l.currentDatabase, l.currentSchema, false /* caseSensitive */)-- l.changedResources.AddTable(- d,- s,- &storepb.ChangedResourceTable{- Name: t,- },- true)+ addDML := func(text string) {+ dmlCount+++ if len(sampleDMLs) < common.MaximumLintExplainSize {+ sampleDMLs = append(sampleDMLs, trimStatement(text))+ }}-}-func (l *tsqlChangedResourceExtractListener) EnterCreate_index(ctx *parser.Create_indexContext) {- tableName := ctx.Table_name()- if tableName == nil {- return+ addTable := func(ref *ast.TableRef, affectData bool) {+ d, s, t := omniTableRefTriple(ref, currentDatabase, currentSchema)+ if t == "" {+ return+ }+ changedResources.AddTable(d, s, &storepb.ChangedResourceTable{Name: t}, affectData)}- d, s, t := normalizeTableNameSeparated(tableName, l.currentDatabase, l.currentSchema, false /* caseSensitive */)-- l.changedResources.AddTable(- d,- s,- &storepb.ChangedResourceTable{- Name: t,- },- false)-}-// EnterDrop_index is called when production drop_index is entered.-func (l *tsqlChangedResourceExtractListener) EnterDrop_index(ctx *parser.Drop_indexContext) {- for _, index := range ctx.AllDrop_relational_or_xml_or_spatial_index() {- fullTable, err := NormalizeFullTableName(index.Full_table_name())- if err != nil {- continue- }- d, _ := NormalizeTSQLIdentifierText(l.currentDatabase)- s, _ := NormalizeTSQLIdentifierText(l.currentSchema)- if fullTable.Database != "" {- d = fullTable.Database+ for _, unifiedAST := range asts {+ omniAST, ok := unifiedAST.(*OmniAST)+ if !ok {+ return nil, errors.New("expected OmniAST for MSSQL")}- if fullTable.Schema != "" {- s = fullTable.Schema+ if omniAST.Node == nil {+ continue}- l.changedResources.AddTable(- d,- s,- &storepb.ChangedResourceTable{- Name: fullTable.Table,- },- false)- }-}--func (l *tsqlChangedResourceExtractListener) EnterInsert_statement(ctx *parser.Insert_statementContext) {- if ctx.Ddl_object() != nil && ctx.Ddl_object().Full_table_name() != nil {- table, err := NormalizeFullTableName(ctx.Ddl_object().Full_table_name())- if err == nil && table != nil && table.Table != "" {- d := table.Database- if d == "" {- d = l.currentDatabase+ switch n := omniAST.Node.(type) {+ case *ast.CreateTableStmt:+ addTable(n.Name, false)++ case *ast.AlterTableStmt:+ addTable(n.Name, true)++ case *ast.DropStmt:+ switch n.ObjectType {+ case ast.DropTable:+ if n.Names == nil {+ continue+ }+ for _, item := range n.Names.Items {+ if ref, ok := item.(*ast.TableRef); ok {+ addTable(ref, true)+ }+ }+ case ast.DropIndex:+ if n.OnTables == nil {+ continue+ }+ for _, item := range n.OnTables.Items {+ if ref, ok := item.(*ast.TableRef); ok {+ addTable(ref, false)+ }+ }+ default:}- s := table.Schema- if s == "" {- s = l.currentSchema- }- l.changedResources.AddTable(- d,- s,- &storepb.ChangedResourceTable{- Name: table.Table,- },- false,- )- }- }- if ctx.Insert_statement_value() != nil && ctx.Insert_statement_value().Derived_table() != nil && ctx.Insert_statement_value().Derived_table().Table_value_constructor() != nil {- tvc := ctx.Insert_statement_value().Derived_table().Table_value_constructor()- l.insertCount += len(tvc.AllExpression_list_())- return- }+ case *ast.CreateIndexStmt:+ addTable(n.Table, false)- // Track DMLs.- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, trimStatement(l.text))- }-}+ case *ast.InsertStmt:+ if ref, ok := n.Relation.(*ast.TableRef); ok {+ addTable(ref, false)+ }+ if n.DefaultValues {+ insertCount+++ continue+ }+ if vc, ok := n.Source.(*ast.ValuesClause); ok {+ insertCount += vc.Rows.Len()+ continue+ }+ addDML(omniAST.Text)-func (l *tsqlChangedResourceExtractListener) EnterUpdate_statement(ctx *parser.Update_statementContext) {- if ctx.Ddl_object() != nil && ctx.Ddl_object().Full_table_name() != nil {- table, err := NormalizeFullTableName(ctx.Ddl_object().Full_table_name())- if err == nil && table != nil && table.Table != "" {- d := table.Database- if d == "" {- d = l.currentDatabase+ case *ast.UpdateStmt:+ if ref, ok := n.Relation.(*ast.TableRef); ok {+ addTable(ref, false)}- s := table.Schema- if s == "" {- s = l.currentSchema+ addDML(omniAST.Text)++ case *ast.DeleteStmt:+ if ref, ok := n.Relation.(*ast.TableRef); ok {+ addTable(ref, false)}- l.changedResources.AddTable(- d,- s,- &storepb.ChangedResourceTable{- Name: table.Table,- },- false,- )+ addDML(omniAST.Text)++ default:}}- // Track DMLs.- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, trimStatement(l.text))- }+ return &base.ChangeSummary{+ ChangedResources: changedResources,+ SampleDMLS: sampleDMLs,+ DMLCount: dmlCount,+ InsertCount: insertCount,+ }, nil}-func (l *tsqlChangedResourceExtractListener) EnterDelete_statement(ctx *parser.Delete_statementContext) {- from := ctx.Delete_statement_from()- if from.Ddl_object() != nil && from.Ddl_object().Full_table_name() != nil {- table, err := NormalizeFullTableName(from.Ddl_object().Full_table_name())- if err == nil && table != nil && table.Table != "" {- d := table.Database- if d == "" {- d = l.currentDatabase- }- s := table.Schema- if s == "" {- s = l.currentSchema- }- l.changedResources.AddTable(- d,- s,- &storepb.ChangedResourceTable{- Name: table.Table,- },- false,- )- }+func omniTableRefTriple(ref *ast.TableRef, defaultDatabase, defaultSchema string) (string, string, string) {+ if ref == nil {+ return defaultDatabase, defaultSchema, ""}-- // Track DMLs.- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, trimStatement(l.text))+ db := ref.Database+ if db == "" {+ db = defaultDatabase+ }+ schema := ref.Schema+ if schema == "" {+ schema = defaultSchema}+ return db, schema, ref.Object}func trimStatement(statement string) string {- // TODO(d): why test is "UPDATE t1 SET c1 = 5\n;".return strings.TrimLeftFunc(strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon), unicode.IsSpace) + ";"}diff --git a/backend/plugin/parser/tsql/resource_change_test.go b/backend/plugin/parser/tsql/resource_change_test.goindex d43fbc160f..54effedb15 100644--- a/backend/plugin/parser/tsql/resource_change_test.go+++ b/backend/plugin/parser/tsql/resource_change_test.go@@ -43,3 +43,30 @@ func TestExtractChangedResources(t *testing.T) {require.NoError(t, err)require.Equal(t, want, got)}++func TestExtractChangedResources_DropIndex(t *testing.T) {+ statement := `DROP INDEX idx1 ON t1;`++ want := model.NewChangedResources(nil)+ want.AddTable("DB", "dbo", &storepb.ChangedResourceTable{Name: "t1"}, false)++ stmts, err := base.ParseStatements(storepb.Engine_MSSQL, statement)+ require.NoError(t, err)+ asts := base.ExtractASTs(stmts)+ got, err := extractChangedResources("DB", "dbo", nil, asts, statement)+ require.NoError(t, err)+ require.Equal(t, want, got.ChangedResources)+}++func TestExtractChangedResources_InsertDefaultValues(t *testing.T) {+ statement := `INSERT INTO t1 DEFAULT VALUES;`++ stmts, err := base.ParseStatements(storepb.Engine_MSSQL, statement)+ require.NoError(t, err)+ asts := base.ExtractASTs(stmts)+ got, err := extractChangedResources("DB", "dbo", nil, asts, statement)+ require.NoError(t, err)+ require.Equal(t, 1, got.InsertCount)+ require.Equal(t, 0, got.DMLCount)+ require.Empty(t, got.SampleDMLS)+}diff --git a/go.mod b/go.modindex 02ce0dbcab..e85840c443 100644--- a/go.mod+++ b/go.mod@@ -34,7 +34,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0- github.com/bytebase/omni v0.0.0-20260417040020-724d3998950a+ github.com/bytebase/omni v0.0.0-20260420093538-51d40fb25301github.com/caarlos0/env/v11 v11.4.0github.com/cenkalti/backoff/v5 v5.0.3github.com/cockroachdb/cockroachdb-parser v0.25.2diff --git a/go.sum b/go.sumindex 03f7ca6619..e08c8f458e 100644--- a/go.sum+++ b/go.sum@@ -251,8 +251,8 @@ github.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888 h1:E8tz8maxpih/vtgithub.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888/go.mod h1:IgLUOjyiHehdE0G3C92IjGYu9lJQgkPwf2mdNSvPWq8=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/omni v0.0.0-20260417040020-724d3998950a h1:jSeJxacP5ZJzvIZwddyX5N0OP9hR87SOGnht3HdX9B4=-github.com/bytebase/omni v0.0.0-20260417040020-724d3998950a/go.mod h1:AT+iLKCu7NyxTaYaOLzETRFBVurjRaqUU3YmFron9pw=+github.com/bytebase/omni v0.0.0-20260420093538-51d40fb25301 h1:uEoT91OLtXfTdTV2B/iGfr4cDGDRpoAk7eIXl4HhWZk=+github.com/bytebase/omni v0.0.0-20260420093538-51d40fb25301/go.mod h1:AT+iLKCu7NyxTaYaOLzETRFBVurjRaqUU3YmFron9pw=github.com/bytebase/parser v0.0.0-20260417075056-57b6ef7a2640 h1:g4Jed1/Jv/DHBiQmEgN5ILOQDDBjFlG4N4Lp+3B4aYE=github.com/bytebase/parser v0.0.0-20260417075056-57b6ef7a2640/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=