Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(advisor/tidb): pilot omni AST migration (Phase 1.5 §1.5.1) (#20140)
Vincent Huang94035b526219
diff --git a/backend/plugin/advisor/advisor.go b/backend/plugin/advisor/advisor.goindex 8ca046a65d..594ef8f770 100644--- a/backend/plugin/advisor/advisor.go+++ b/backend/plugin/advisor/advisor.go@@ -68,6 +68,51 @@ type Context struct {// Used for test only.NoAppendBuiltin bool++ // memo is a per-review key/value cache shared across rules in one+ // SQLReviewCheck invocation. Initialized once by SQLReviewCheck before the+ // rule loop; the map header propagates by value across rule invocations+ // (advisors run sequentially, no synchronization needed).+ //+ // Engine-specific helpers use this to amortize work that would otherwise+ // repeat per advisor, e.g. omni re-parsing during the tidb advisor+ // migration (advisor/tidb/utils.go).+ //+ // Callers that bypass SQLReviewCheck and construct Context directly+ // without initializing memo will see Memo/SetMemo silently no-op (cache+ // disabled, work repeats). Test fixtures and ad-hoc callers should+ // initialize memo if they want caching behavior.+ memo map[string]any+}++// Memo returns the value previously stored under key.+// Returns (nil, false) on cache miss or uninitialized cache.+func (c Context) Memo(key string) (any, bool) {+ if c.memo == nil {+ return nil, false+ }+ v, ok := c.memo[key]+ return v, ok+}++// SetMemo stores a value under key for retrieval by Memo.+// No-op if the cache is uninitialized.+func (c Context) SetMemo(key string, value any) {+ if c.memo == nil {+ return+ }+ c.memo[key] = value+}++// InitMemo initializes the per-review memo cache if not already initialized.+// Called by SQLReviewCheck before its rule loop. Tests that bypass+// SQLReviewCheck and want caching behavior should call this on the Context+// they construct (Context is passed by value through the rule loop, so the+// init must happen on the original before any copies are made).+func (c *Context) InitMemo() {+ if c.memo == nil {+ c.memo = make(map[string]any)+ }}// Advisor is the interface for advisor.diff --git a/backend/plugin/advisor/sql_review.go b/backend/plugin/advisor/sql_review.goindex be0413fdfa..5ca458541a 100644--- a/backend/plugin/advisor/sql_review.go+++ b/backend/plugin/advisor/sql_review.go@@ -134,6 +134,11 @@ func SQLReviewCheck(}}+ // Initialize the per-review memo so engine-specific helpers can amortize+ // work across rules (e.g. omni re-parsing in advisor/tidb). The map header+ // propagates by value across rules; the underlying map is shared.+ checkContext.memo = make(map[string]any)+var errorAdvices, warningAdvices []*storepb.Advicefor _, rule := range ruleList {if rule.Engine != storepb.Engine_ENGINE_UNSPECIFIED && rule.Engine != checkContext.DBType {diff --git a/backend/plugin/advisor/tidb/advisor_column_no_null.go b/backend/plugin/advisor/tidb/advisor_column_no_null.goindex f7ffb1759e..1d6f664726 100644--- a/backend/plugin/advisor/tidb/advisor_column_no_null.go+++ b/backend/plugin/advisor/tidb/advisor_column_no_null.go@@ -5,7 +5,7 @@ import ("fmt""slices"- "github.com/pingcap/tidb/pkg/parser/ast"+ "github.com/bytebase/omni/tidb/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -16,7 +16,6 @@ import (var (_ advisor.Advisor = (*ColumnNoNullAdvisor)(nil)- _ ast.Visitor = (*columnNoNullChecker)(nil))func init() {@@ -29,8 +28,7 @@ type ColumnNoNullAdvisor struct {// Check checks for column no NULL value.func (*ColumnNoNullAdvisor) Check(_ context.Context, checkCtx advisor.Context) ([]*storepb.Advice, error) {- root, err := getTiDBNodes(checkCtx)-+ stmts, err := getTiDBOmniNodes(checkCtx)if err != nil {return nil, err}@@ -39,6 +37,7 @@ func (*ColumnNoNullAdvisor) Check(_ context.Context, checkCtx advisor.Context) (if err != nil {return nil, err}+checker := &columnNoNullChecker{level: level,title: checkCtx.Rule.Type.String(),@@ -46,8 +45,14 @@ func (*ColumnNoNullAdvisor) Check(_ context.Context, checkCtx advisor.Context) (finalMetadata: checkCtx.FinalMetadata,}- for _, stmtNode := range root {- (stmtNode).Accept(checker)+ for _, ostmt := range stmts {+ switch n := ostmt.Node.(type) {+ case *ast.CreateTableStmt:+ checker.checkCreateTable(n, ostmt)+ case *ast.AlterTableStmt:+ checker.checkAlterTable(n, ostmt)+ default:+ }}return checker.generateAdvice(), nil@@ -61,9 +66,80 @@ type columnNoNullChecker struct {finalMetadata *model.DatabaseMetadata}-func (checker *columnNoNullChecker) generateAdvice() []*storepb.Advice {+func (c *columnNoNullChecker) checkCreateTable(n *ast.CreateTableStmt, ostmt OmniStmt) {+ if n.Table == nil {+ return+ }+ tableName := n.Table.Name+ for _, col := range n.Columns {+ if col == nil {+ continue+ }+ cn := columnName{+ tableName: tableName,+ columnName: col.Name,+ line: ostmt.AbsoluteLine(col.Loc.Start),+ }+ if _, exists := c.columnSet[cn.name()]; !exists {+ c.columnSet[cn.name()] = cn+ }+ }+}++func (c *columnNoNullChecker) checkAlterTable(n *ast.AlterTableStmt, ostmt OmniStmt) {+ if n.Table == nil {+ return+ }+ tableName := n.Table.Name+ // Match prior pingcap behavior: report the line of the ALTER TABLE statement+ // itself, not the line of each affected column inside the statement.+ stmtLine := ostmt.AbsoluteLine(n.Loc.Start)+ for _, cmd := range n.Commands {+ if cmd == nil {+ continue+ }+ var columns []string+ switch cmd.Type {+ case ast.ATAddColumn:+ // omni populates either Columns (multi-column ADD COLUMN (...))+ // or Column (single-column ADD COLUMN). Mutually exclusive in+ // practice but not enforced by the type — read defensively.+ if len(cmd.Columns) > 0 {+ for _, col := range cmd.Columns {+ if col != nil {+ columns = append(columns, col.Name)+ }+ }+ } else if cmd.Column != nil {+ columns = append(columns, cmd.Column.Name)+ }+ case ast.ATChangeColumn:+ // Only care about the new column name.+ if cmd.Column != nil {+ columns = append(columns, cmd.Column.Name)+ }+ default:+ // MODIFY COLUMN (ATModifyColumn) is intentionally NOT handled.+ // Pre-existing inheritance from the pingcap-AST version of this+ // advisor — `ALTER TABLE t MODIFY COLUMN x INT NULL` slips past.+ // Not an omni-introduced regression; preserve current behavior.+ }+ for _, name := range columns {+ cn := columnName{+ tableName: tableName,+ columnName: name,+ line: stmtLine,+ }+ if _, exists := c.columnSet[cn.name()]; !exists {+ c.columnSet[cn.name()] = cn+ }+ }+ }+}++func (c *columnNoNullChecker) generateAdvice() []*storepb.Advice {var columnList []columnName- for _, column := range checker.columnSet {+ for _, column := range c.columnSet {columnList = append(columnList, column)}slices.SortFunc(columnList, func(i, j columnName) int {@@ -83,7 +159,7 @@ func (checker *columnNoNullChecker) generateAdvice() []*storepb.Advice {})for _, column := range columnList {- schema := checker.finalMetadata.GetSchemaMetadata("")+ schema := c.finalMetadata.GetSchemaMetadata("")if schema == nil {continue}@@ -93,17 +169,17 @@ func (checker *columnNoNullChecker) generateAdvice() []*storepb.Advice {}col := table.GetColumn(column.columnName)if col != nil && col.GetProto().Nullable {- checker.adviceList = append(checker.adviceList, &storepb.Advice{- Status: checker.level,+ c.adviceList = append(c.adviceList, &storepb.Advice{+ Status: c.level,Code: code.ColumnCannotNull.Int32(),- Title: checker.title,+ Title: c.title,Content: fmt.Sprintf("`%s`.`%s` cannot have NULL value", column.tableName, column.columnName),StartPosition: common.ConvertANTLRLineToPosition(column.line),})}}- return checker.adviceList+ return c.adviceList}type columnName struct {@@ -115,68 +191,3 @@ type columnName struct {func (c columnName) name() string {return fmt.Sprintf("%s.%s", c.tableName, c.columnName)}--// Enter implements the ast.Visitor interface.-func (checker *columnNoNullChecker) Enter(in ast.Node) (ast.Node, bool) {- switch node := in.(type) {- // CREATE TABLE- case *ast.CreateTableStmt:- for _, column := range node.Cols {- col := columnName{- tableName: node.Table.Name.O,- columnName: column.Name.Name.O,- line: column.OriginTextPosition(),- }- if _, exists := checker.columnSet[col.name()]; !exists {- checker.columnSet[col.name()] = col- }- }- // ALTER TABLE- case *ast.AlterTableStmt:- for _, spec := range node.Specs {- switch spec.Tp {- // ADD COLUMNS- case ast.AlterTableAddColumns:- for _, column := range spec.NewColumns {- col := columnName{- tableName: node.Table.Name.O,- columnName: column.Name.Name.O,- line: node.OriginTextPosition(),- }- if _, exists := checker.columnSet[col.name()]; !exists {- checker.columnSet[col.name()] = col- }- }- // CHANGE COLUMN- case ast.AlterTableChangeColumn:- col := columnName{- tableName: node.Table.Name.O,- columnName: spec.NewColumns[0].Name.Name.O,- line: node.OriginTextPosition(),- }- if _, exists := checker.columnSet[col.name()]; !exists {- checker.columnSet[col.name()] = col- }- default:- // Skip other alter table specification types- }- }- default:- }-- return in, false-}--// Leave implements the ast.Visitor interface.-func (*columnNoNullChecker) Leave(in ast.Node) (ast.Node, bool) {- return in, true-}--func canNull(column *ast.ColumnDef) bool {- for _, option := range column.Options {- if option.Tp == ast.ColumnOptionNotNull || option.Tp == ast.ColumnOptionPrimaryKey {- return false- }- }- return true-}diff --git a/backend/plugin/advisor/tidb/advisor_insert_must_specify_column.go b/backend/plugin/advisor/tidb/advisor_insert_must_specify_column.goindex 8f8ff9e209..cfa3004310 100644--- a/backend/plugin/advisor/tidb/advisor_insert_must_specify_column.go+++ b/backend/plugin/advisor/tidb/advisor_insert_must_specify_column.go@@ -1,12 +1,10 @@package tidb-// Framework code is generated by the generator.-import ("context""fmt"- "github.com/pingcap/tidb/pkg/parser/ast"+ "github.com/bytebase/omni/tidb/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -16,7 +14,6 @@ import (var (_ advisor.Advisor = (*InsertMustSpecifyColumnAdvisor)(nil)- _ ast.Visitor = (*insertMustSpecifyColumnChecker)(nil))func init() {@@ -29,8 +26,7 @@ type InsertMustSpecifyColumnAdvisor struct {// Check checks for to enforce column specified.func (*InsertMustSpecifyColumnAdvisor) Check(_ context.Context, checkCtx advisor.Context) ([]*storepb.Advice, error) {- stmtList, err := getTiDBNodes(checkCtx)-+ stmts, err := getTiDBOmniNodes(checkCtx)if err != nil {return nil, err}@@ -39,15 +35,14 @@ func (*InsertMustSpecifyColumnAdvisor) Check(_ context.Context, checkCtx advisorif err != nil {return nil, err}+checker := &insertMustSpecifyColumnChecker{level: level,title: checkCtx.Rule.Type.String(),}- for _, stmt := range stmtList {- checker.text = stmt.Text()- checker.line = stmt.OriginTextPosition()- (stmt).Accept(checker)+ for _, ostmt := range stmts {+ checker.checkStmt(ostmt)}return checker.adviceList, nil@@ -57,28 +52,33 @@ type insertMustSpecifyColumnChecker struct {adviceList []*storepb.Advicelevel storepb.Advice_Statustitle string- text string- line int}-// Enter implements the ast.Visitor interface.-func (checker *insertMustSpecifyColumnChecker) Enter(in ast.Node) (ast.Node, bool) {- if node, ok := in.(*ast.InsertStmt); ok {- if node.Columns == nil {- checker.adviceList = append(checker.adviceList, &storepb.Advice{- Status: checker.level,- Code: code.InsertNotSpecifyColumn.Int32(),- Title: checker.title,- Content: fmt.Sprintf("The INSERT statement must specify columns but \"%s\" does not", checker.text),- StartPosition: common.ConvertANTLRLineToPosition(checker.line),- })- }+func (c *insertMustSpecifyColumnChecker) checkStmt(ostmt OmniStmt) {+ node, ok := ostmt.Node.(*ast.InsertStmt)+ if !ok {+ return}-- return in, false-}--// Leave implements the ast.Visitor interface.-func (*insertMustSpecifyColumnChecker) Leave(in ast.Node) (ast.Node, bool) {- return in, true+ // Already has an explicit column list — no advice.+ if len(node.Columns) > 0 {+ return+ }+ // Mirror mysql/rule_insert_must_specify_column.go: flag only the VALUES+ // and SELECT forms when they lack an explicit column list. INSERT ... SET+ // (and equivalent REPLACE ... SET, TABLE-source, DEFAULT VALUES, etc.)+ // specify their target columns through assignments or source semantics+ // and were not flagged by the prior pingcap-AST rule. Without this carve-+ // out the omni-migrated rule false-positives on every SET-form insert.+ isValuesForm := len(node.Values) > 0+ isSelectForm := node.Select != nil+ if !isValuesForm && !isSelectForm {+ return+ }+ c.adviceList = append(c.adviceList, &storepb.Advice{+ Status: c.level,+ Code: code.InsertNotSpecifyColumn.Int32(),+ Title: c.title,+ Content: fmt.Sprintf("The INSERT statement must specify columns but \"%s\" does not", ostmt.Text),+ StartPosition: common.ConvertANTLRLineToPosition(ostmt.AbsoluteLine(node.Loc.Start)),+ })}diff --git a/backend/plugin/advisor/tidb/test/statement_insert_must_specify_column.yaml b/backend/plugin/advisor/tidb/test/statement_insert_must_specify_column.yamlindex 945800b0c6..2a3a722b08 100644--- a/backend/plugin/advisor/tidb/test/statement_insert_must_specify_column.yaml+++ b/backend/plugin/advisor/tidb/test/statement_insert_must_specify_column.yaml@@ -11,3 +11,20 @@line: 1column: 0endposition: null+- statement: INSERT INTO tech_book SET id = 1, name = '1'+ changeType: 1+- statement: REPLACE INTO tech_book SET id = 1, name = '1'+ changeType: 1+- statement: INSERT INTO tech_book(id, name) SELECT id, name FROM other_book+ changeType: 1+- statement: INSERT INTO tech_book SELECT id, name FROM other_book+ changeType: 1+ want:+ - status: 2+ code: 1107+ title: STATEMENT_INSERT_MUST_SPECIFY_COLUMN+ content: The INSERT statement must specify columns but "INSERT INTO tech_book SELECT id, name FROM other_book" does not+ startposition:+ line: 1+ column: 0+ endposition: nulldiff --git a/backend/plugin/advisor/tidb/utils.go b/backend/plugin/advisor/tidb/utils.goindex bf894f1491..0ad059e531 100644--- a/backend/plugin/advisor/tidb/utils.go+++ b/backend/plugin/advisor/tidb/utils.go@@ -2,6 +2,7 @@package tidbimport (+ "log/slog""slices""strings"@@ -11,6 +12,8 @@ import ("github.com/pingcap/tidb/pkg/parser/types""github.com/pkg/errors"+ omniast "github.com/bytebase/omni/tidb/ast"+"github.com/bytebase/bytebase/backend/plugin/advisor"tidbparser "github.com/bytebase/bytebase/backend/plugin/parser/tidb")@@ -99,3 +102,106 @@ func getTiDBNodes(checkCtx advisor.Context) ([]ast.StmtNode, error) {}return stmtNodes, nil}++// OmniStmt bundles an omni/tidb AST node with the source text and base line of+// the statement it came from. The base line is needed to convert byte offsets+// inside Text into absolute line numbers in the original full SQL.+type OmniStmt struct {+ Node omniast.Node+ Text string+ BaseLine int // 0-based line index of the first line of Text in the original SQL+}++// AbsoluteLine converts a byte offset within s.Text into a 1-based line number+// in the original SQL.+func (s OmniStmt) AbsoluteLine(byteOffset int) int {+ pos := tidbparser.ByteOffsetToRunePosition(s.Text, byteOffset)+ return s.BaseLine + int(pos.Line)+}++// canNull reports whether the given pingcap-AST column may have NULL values+// (i.e. the column has no NOT NULL or PRIMARY KEY constraint).+//+// Moved here from advisor_column_no_null.go during its migration to omni AST.+// Still consumed by advisor_column_set_default_for_not_null.go (un-migrated).+// Delete when set_default_for_not_null migrates to omni AST.+// Tracked: https://linear.app/bytebase/issue/BYT-9362+func canNull(column *ast.ColumnDef) bool {+ for _, option := range column.Options {+ if option.Tp == ast.ColumnOptionNotNull || option.Tp == ast.ColumnOptionPrimaryKey {+ return false+ }+ }+ return true+}++// omniStmtsCacheKey is the advisor.Context.Memo key for the per-review+// omni-parse result. All migrated tidb advisors share one parse pass through+// this cache.+const omniStmtsCacheKey = "tidb.omniStmts"++// getTiDBOmniNodes extracts omni/tidb AST nodes from the advisor context by+// re-parsing each statement's text with omni. Used by advisors during the+// migration off the native pingcap parser.+//+// While the registered ParseStatementsFunc still returns native pingcap ASTs+// (preserved by Phase 1.5 for backward compat with un-migrated advisors), this+// helper re-parses with omni. Migrated advisors call this; un-migrated+// advisors continue to call getTiDBNodes.+//+// Phase 1.5 invariants enforced here:+//+// - Single-parse-per-review: result is cached on advisor.Context.Memo so+// subsequent advisors in the same review reuse the parse work. Cost stays+// 1× regardless of how many advisors migrate.+// - Soft-fail on omni grammar gaps: a statement that fails to parse with+// omni is logged and skipped. Other statements (and other advisors) keep+// working. This protects review continuity while omni grammar catches up+// to pingcap on Tier 4 deferred features (FLASHBACK, SEQUENCE, BATCH DML,+// etc. — see plans/2026-04-23-omni-tidb-completion-plan.md §Phase 2).+//+// After all advisors are migrated and the dispatcher is flipped to omni, this+// helper can be simplified to read directly from checkCtx.ParsedStatements+// without re-parsing.+func getTiDBOmniNodes(checkCtx advisor.Context) ([]OmniStmt, error) {+ if cached, ok := checkCtx.Memo(omniStmtsCacheKey); ok {+ if stmts, typeOK := cached.([]OmniStmt); typeOK {+ return stmts, nil+ }+ }++ if checkCtx.ParsedStatements == nil {+ return nil, errors.New("ParsedStatements is not provided in context")+ }++ var result []OmniStmt+ for _, stmt := range checkCtx.ParsedStatements {+ if stmt.Empty {+ continue+ }+ list, err := tidbparser.ParseTiDBOmni(stmt.Text)+ if err != nil {+ // Soft-fail: omni may not yet support every TiDB grammar feature+ // pingcap accepts. Skip this statement; advisors emit no advice+ // for it but the review continues. Promote to higher log level+ // when this signals a real omni regression vs. an expected gap.+ slog.Debug("omni/tidb parse failed; skipping statement for omni-aware advisors",+ slog.String("error", err.Error()),+ )+ continue+ }+ if list == nil {+ continue+ }+ for _, item := range list.Items {+ result = append(result, OmniStmt{+ Node: item,+ Text: stmt.Text,+ BaseLine: stmt.BaseLine(),+ })+ }+ }++ checkCtx.SetMemo(omniStmtsCacheKey, result)+ return result, nil+}diff --git a/backend/plugin/advisor/tidb/utils_test.go b/backend/plugin/advisor/tidb/utils_test.gonew file mode 100644index 0000000000..a646cf7e24--- /dev/null+++ b/backend/plugin/advisor/tidb/utils_test.go@@ -0,0 +1,85 @@+package tidb++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// TestGetTiDBOmniNodesCachesAcrossRules pins the Phase 1.5 single-parse-per-+// review invariant: when multiple migrated advisors call getTiDBOmniNodes+// with the same advisor.Context, omni parsing happens once and the result+// slice is reused. Without this, parse cost scales with migrated-advisor+// count and review latency degrades monotonically across the migration+// window. See plans/2026-04-23-omni-tidb-completion-plan.md §1.5.0.+func TestGetTiDBOmniNodesCachesAcrossRules(t *testing.T) {+ ctx := advisor.Context{+ ParsedStatements: []base.ParsedStatement{+ {+ Statement: base.Statement{+ Text: "CREATE TABLE t (id INT)",+ Start: &storepb.Position{Line: 1},+ },+ },+ },+ }+ ctx.InitMemo()++ first, err := getTiDBOmniNodes(ctx)+ require.NoError(t, err)+ require.NotEmpty(t, first, "first call should return a parsed statement")++ second, err := getTiDBOmniNodes(ctx)+ require.NoError(t, err)+ require.NotEmpty(t, second)++ // Cache hit: same backing array returned both calls. Address equality+ // on the first element proves the slice header was reused, not a fresh+ // re-parse that would allocate a new []OmniStmt.+ require.True(t, &first[0] == &second[0],+ "expected cached []OmniStmt; got fresh re-parse — single-parse-per-review invariant broken")+}++// TestGetTiDBOmniNodesSoftFailsOnGrammarGap pins the Phase 1.5 soft-fail+// invariant: a statement that fails to parse with omni is logged and+// skipped, not propagated as an error that breaks the advisor. The review+// continues with whatever statements omni did parse.+//+// This test feeds a statement that is valid TiDB SQL but uses a deferred+// Phase 2 grammar feature (BATCH ... DRY RUN ...) that omni/tidb does not+// yet support. The expectation is that getTiDBOmniNodes returns the+// successfully-parsed statements and skips the BATCH one, with no error.+func TestGetTiDBOmniNodesSoftFailsOnGrammarGap(t *testing.T) {+ ctx := advisor.Context{+ ParsedStatements: []base.ParsedStatement{+ {+ Statement: base.Statement{+ Text: "CREATE TABLE t (id INT)",+ Start: &storepb.Position{Line: 1},+ },+ },+ {+ Statement: base.Statement{+ Text: "BATCH ON id LIMIT 5000 UPDATE t SET v = v + 1",+ Start: &storepb.Position{Line: 2},+ },+ },+ {+ Statement: base.Statement{+ Text: "INSERT INTO t (id) VALUES (1)",+ Start: &storepb.Position{Line: 3},+ },+ },+ },+ }+ ctx.InitMemo()++ got, err := getTiDBOmniNodes(ctx)+ require.NoError(t, err, "soft-fail invariant: parse errors must not propagate")+ require.GreaterOrEqual(t, len(got), 2,+ "expected at least the two non-BATCH statements to parse and be returned")+}