Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(advisor/tidb): migrate column_set_default_for_not_null + column_disallow_set_charset to omni AST (Phase 1.5 §1.5.3 batch 7) (#20223)
Vincent Huangeeec0226de7a
diff --git a/backend/plugin/advisor/tidb/advisor_column_disallow_set_charset.go b/backend/plugin/advisor/tidb/advisor_column_disallow_set_charset.goindex 16fc2d7c00..579254bbc1 100644--- a/backend/plugin/advisor/tidb/advisor_column_disallow_set_charset.go+++ b/backend/plugin/advisor/tidb/advisor_column_disallow_set_charset.go@@ -1,125 +1,120 @@package tidb-// Framework code is generated by the generator.-import ("context""fmt"- advisorcode "github.com/bytebase/bytebase/backend/plugin/advisor/code"-- "github.com/pingcap/tidb/pkg/parser/ast"+ "github.com/bytebase/omni/tidb/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor"+ advisorcode "github.com/bytebase/bytebase/backend/plugin/advisor/code")var (_ advisor.Advisor = (*ColumnDisallowSetCharsetAdvisor)(nil)- _ ast.Visitor = (*columnDisallowSetCharsetChecker)(nil))func init() {advisor.Register(storepb.Engine_TIDB, storepb.SQLReviewRule_COLUMN_DISALLOW_SET_CHARSET, &ColumnDisallowSetCharsetAdvisor{})}-// ColumnDisallowSetCharsetAdvisor is the advisor checking for disallow set column charset.+// ColumnDisallowSetCharsetAdvisor checks for disallowed column-level+// CHARACTER SET clauses.type ColumnDisallowSetCharsetAdvisor struct {}-// Check checks for disallow set column charset.+// Check is Recipe A. Same wrapper-safety rationale as the batch-6+// charset/collation/comment family.+//+// Cardinality: per-statement, single-advice with first-violation-wins.+// Pingcap's pattern set `code` and broke out of inner loops on first+// violation, with a single append after the visitor switch. Mirroring+// that semantic via early-exit `break` in the column loop / `done` flag+// for ALTER's outer spec loop.+//+// Closes BYT-9412 (getColumnCharset cleanup) by being the last consumer+// of the pingcap-typed getColumnCharset helper.func (*ColumnDisallowSetCharsetAdvisor) Check(_ context.Context, checkCtx advisor.Context) ([]*storepb.Advice, error) {- stmtList, err := getTiDBNodes(checkCtx)-+ stmts, err := getTiDBOmniNodes(checkCtx)if err != nil {return nil, err}-level, err := advisor.NewStatusBySQLReviewRuleLevel(checkCtx.Rule.Level)if err != nil {return nil, err}- checker := &columnDisallowSetCharsetChecker{- level: level,- title: checkCtx.Rule.Type.String(),- }-- for _, stmt := range stmtList {- checker.text = stmt.Text()- checker.line = stmt.OriginTextPosition()- (stmt).Accept(checker)- }- return checker.adviceList, nil-}--type columnDisallowSetCharsetChecker struct {- adviceList []*storepb.Advice- level storepb.Advice_Status- title string- text string- line int-}+ title := checkCtx.Rule.Type.String()+ var adviceList []*storepb.Advice-// Enter implements the ast.Visitor interface.-func (checker *columnDisallowSetCharsetChecker) Enter(in ast.Node) (ast.Node, bool) {- code := advisorcode.Ok- switch node := in.(type) {- case *ast.CreateTableStmt:- for _, column := range node.Cols {- charset := getColumnCharset(column)- if !checkCharset(charset) {- code = advisorcode.SetColumnCharset- break+ for _, ostmt := range stmts {+ var found bool+ switch n := ostmt.Node.(type) {+ case *ast.CreateTableStmt:+ for _, col := range n.Columns {+ if col == nil {+ continue+ }+ if !columnCharsetAllowed(col) {+ found = true+ break // pingcap parity: first-violating-column wins+ }}- }- case *ast.AlterTableStmt:- for _, spec := range node.Specs {- switch spec.Tp {- case ast.AlterTableAddColumns:- for _, column := range spec.NewColumns {- charset := getColumnCharset(column)- if !checkCharset(charset) {- code = advisorcode.SetColumnCharset+ case *ast.AlterTableStmt:+ for _, cmd := range n.Commands {+ if cmd == nil {+ continue+ }+ switch cmd.Type {+ case ast.ATAddColumn:+ for _, col := range addColumnTargets(cmd) {+ if col == nil {+ continue+ }+ if !columnCharsetAllowed(col) {+ found = true+ break+ }+ }+ case ast.ATChangeColumn, ast.ATModifyColumn:+ if cmd.Column != nil && !columnCharsetAllowed(cmd.Column) {+ found = true}+ default:}- case ast.AlterTableChangeColumn, ast.AlterTableModifyColumn:- charset := getColumnCharset(spec.NewColumns[0])- if !checkCharset(charset) {- code = advisorcode.SetColumnCharset+ if found {+ // pingcap parity: outer specs loop breaks on first+ // violating spec.+ break}- default:- // Other alter table types- }- if code != advisorcode.Ok {- break}+ default:}- default:- }- if code != advisorcode.Ok {- checker.adviceList = append(checker.adviceList, &storepb.Advice{- Status: checker.level,+ if !found {+ continue+ }+ adviceList = append(adviceList, &storepb.Advice{+ Status: level,Code: advisorcode.SetColumnCharset.Int32(),- Title: checker.title,- Content: fmt.Sprintf("Disallow set column charset but \"%s\" does", checker.text),- StartPosition: common.ConvertANTLRLineToPosition(checker.line),+ Title: title,+ Content: fmt.Sprintf("Disallow set column charset but \"%s\" does", ostmt.TrimmedText()),+ StartPosition: common.ConvertANTLRLineToPosition(ostmt.FirstTokenLine()),})}- return in, false-}--// Leave implements the ast.Visitor interface.-func (*columnDisallowSetCharsetChecker) Leave(in ast.Node) (ast.Node, bool) {- return in, true+ return adviceList, nil}-func checkCharset(charset string) bool {- switch charset {- // empty charset or binary for JSON.+// columnCharsetAllowed reports whether a column's CHARACTER SET is+// acceptable to this rule. Mirrors pingcap's checkCharset: empty (no+// clause) and "binary" (used for JSON-like types) are allowed; everything+// else is a violation.+func columnCharsetAllowed(col *ast.ColumnDef) bool {+ cs := omniColumnCharset(col)+ switch cs {case "", "binary":return truedefault:diff --git a/backend/plugin/advisor/tidb/advisor_column_set_default_for_not_null.go b/backend/plugin/advisor/tidb/advisor_column_set_default_for_not_null.goindex d3c06f612c..52be7f1a86 100644--- a/backend/plugin/advisor/tidb/advisor_column_set_default_for_not_null.go+++ b/backend/plugin/advisor/tidb/advisor_column_set_default_for_not_null.go@@ -1,147 +1,178 @@package tidb-// Framework code is generated by the generator.-import ("context""fmt"- "github.com/bytebase/bytebase/backend/plugin/advisor/code"-- "github.com/pingcap/tidb/pkg/parser/ast"+ "github.com/bytebase/omni/tidb/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code")var (_ advisor.Advisor = (*ColumnSetDefaultForNotNullAdvisor)(nil)- _ ast.Visitor = (*columnSetDefaultForNotNullChecker)(nil))func init() {advisor.Register(storepb.Engine_TIDB, storepb.SQLReviewRule_COLUMN_SET_DEFAULT_FOR_NOT_NULL, &ColumnSetDefaultForNotNullAdvisor{})}-// ColumnSetDefaultForNotNullAdvisor is the advisor checking for set default value for not null column.+// ColumnSetDefaultForNotNullAdvisor checks for set default value for not null columns.type ColumnSetDefaultForNotNullAdvisor struct {}-// Check checks for set default value for not null column.+// Check is Recipe A. Same wrapper-safety rationale as the batch-6+// charset/collation/comment family: EXPLAIN doesn't accept DDL in either+// pingcap or omni grammar, and CREATE/ALTER TABLE can't nest themselves.+//+// Cardinality: per-violating-column (NOT per-statement). Pingcap's+// pattern was build a list during the visit and append one advice per+// list entry after the switch — preserved here.+//+// Closes BYT-9362 (canNull cleanup) by being the last consumer of the+// pingcap-typed canNull helper.func (*ColumnSetDefaultForNotNullAdvisor) Check(_ context.Context, checkCtx advisor.Context) ([]*storepb.Advice, error) {- stmtList, err := getTiDBNodes(checkCtx)-+ stmts, err := getTiDBOmniNodes(checkCtx)if err != nil {return nil, err}-level, err := advisor.NewStatusBySQLReviewRuleLevel(checkCtx.Rule.Level)if err != nil {return nil, err}- checker := &columnSetDefaultForNotNullChecker{- level: level,- title: checkCtx.Rule.Type.String(),- }-- for _, stmt := range stmtList {- checker.text = stmt.Text()- (stmt).Accept(checker)- }- return checker.adviceList, nil-}--type columnSetDefaultForNotNullChecker struct {- adviceList []*storepb.Advice- level storepb.Advice_Status- title string- text string-}+ title := checkCtx.Rule.Type.String()+ var adviceList []*storepb.Advice-// Enter implements the ast.Visitor interface.-func (checker *columnSetDefaultForNotNullChecker) Enter(in ast.Node) (ast.Node, bool) {- var notNullColumnWithNoDefault []columnName- switch node := in.(type) {- // CREATE TABLE- case *ast.CreateTableStmt:- pkColumn := make(map[string]bool)- for _, cons := range node.Constraints {- if cons.Tp == ast.ConstraintPrimaryKey {- for _, key := range cons.Keys {- pkColumn[key.Column.Name.O] = true- }- }- }-- for _, column := range node.Cols {- isPk := pkColumn[column.Name.Name.O]- if isPk {+ for _, ostmt := range stmts {+ var rows []columnName+ switch n := ostmt.Node.(type) {+ case *ast.CreateTableStmt:+ if n.Table == nil {continue}- if !canNull(column) && !setDefault(column) && needDefault(column) {- notNullColumnWithNoDefault = append(notNullColumnWithNoDefault, columnName{- tableName: node.Table.Name.O,- columnName: column.Name.Name.O,- line: column.OriginTextPosition(),+ tableName := n.Table.Name+ pkCols := omniCreateTablePKColumnSet(n)+ for _, col := range n.Columns {+ if col == nil {+ continue+ }+ if pkCols[col.Name] {+ continue+ }+ if !omniColumnIsNotNull(col) || omniColumnHasDefault(col) || !omniNeedDefault(col) {+ continue+ }+ rows = append(rows, columnName{+ tableName: tableName,+ columnName: col.Name,+ line: ostmt.AbsoluteLine(col.Loc.Start),})}- }- // ALTER TABLE- case *ast.AlterTableStmt:- for _, spec := range node.Specs {- switch spec.Tp {- // ADD COLUMNS- case ast.AlterTableAddColumns:- for _, column := range spec.NewColumns {- if !canNull(column) && !setDefault(column) && needDefault(column) {- notNullColumnWithNoDefault = append(notNullColumnWithNoDefault, columnName{- tableName: node.Table.Name.O,- columnName: column.Name.Name.O,- line: node.OriginTextPosition(),+ case *ast.AlterTableStmt:+ if n.Table == nil {+ continue+ }+ tableName := n.Table.Name+ stmtLine := ostmt.FirstTokenLine()+ for _, cmd := range n.Commands {+ if cmd == nil {+ continue+ }+ switch cmd.Type {+ case ast.ATAddColumn:+ for _, col := range addColumnTargets(cmd) {+ if col == nil {+ continue+ }+ if !omniColumnIsNotNull(col) || omniColumnHasDefault(col) || !omniNeedDefault(col) {+ continue+ }+ rows = append(rows, columnName{+ tableName: tableName,+ columnName: col.Name,+ line: stmtLine,})}- }- // CHANGE COLUMN and MODIFY COLUMN- case ast.AlterTableChangeColumn, ast.AlterTableModifyColumn:- if !canNull(spec.NewColumns[0]) && !setDefault(spec.NewColumns[0]) && needDefault(spec.NewColumns[0]) {- notNullColumnWithNoDefault = append(notNullColumnWithNoDefault, columnName{- tableName: node.Table.Name.O,- columnName: spec.NewColumns[0].Name.Name.O,- line: node.OriginTextPosition(),+ case ast.ATChangeColumn, ast.ATModifyColumn:+ col := cmd.Column+ if col == nil {+ continue+ }+ if !omniColumnIsNotNull(col) || omniColumnHasDefault(col) || !omniNeedDefault(col) {+ continue+ }+ rows = append(rows, columnName{+ tableName: tableName,+ columnName: col.Name,+ line: stmtLine,})+ default:}- default:- // Skip other alter table specification types}+ default:+ continue}- default:- }- for _, column := range notNullColumnWithNoDefault {- checker.adviceList = append(checker.adviceList, &storepb.Advice{- Status: checker.level,- Code: code.NotNullColumnWithNoDefault.Int32(),- Title: checker.title,- Content: fmt.Sprintf("Column `%s`.`%s` is NOT NULL but doesn't have DEFAULT", column.tableName, column.columnName),- StartPosition: common.ConvertANTLRLineToPosition(column.line),- })+ for _, r := range rows {+ adviceList = append(adviceList, &storepb.Advice{+ Status: level,+ Code: code.NotNullColumnWithNoDefault.Int32(),+ Title: title,+ Content: fmt.Sprintf("Column `%s`.`%s` is NOT NULL but doesn't have DEFAULT", r.tableName, r.columnName),+ StartPosition: common.ConvertANTLRLineToPosition(r.line),+ })+ }}- return in, false+ return adviceList, nil}-// Leave implements the ast.Visitor interface.-func (*columnSetDefaultForNotNullChecker) Leave(in ast.Node) (ast.Node, bool) {- return in, true+// omniCreateTablePKColumnSet returns the set of column names declared as+// PRIMARY KEY via table-level constraints in the CREATE TABLE statement+// (e.g. `CREATE TABLE t (id INT, PRIMARY KEY (id))`). Inline column-level+// PRIMARY KEY (e.g. `CREATE TABLE t (id INT PRIMARY KEY)`) is detected+// by omniNeedDefault directly, not by this set.+func omniCreateTablePKColumnSet(n *ast.CreateTableStmt) map[string]bool {+ pkCols := make(map[string]bool)+ for _, c := range n.Constraints {+ if c == nil || c.Type != ast.ConstrPrimaryKey {+ continue+ }+ for _, name := range c.Columns {+ pkCols[name] = true+ }+ }+ return pkCols}-func setDefault(column *ast.ColumnDef) bool {- for _, option := range column.Options {- if option.Tp == ast.ColumnOptionDefaultValue {+// omniColumnIsNotNull reports whether the column's constraint list+// contains a NOT NULL marker (or a PRIMARY KEY marker, which implies NOT+// NULL in MySQL/TiDB semantics). Mirror of pingcap-typed canNull's+// inverse — pingcap returned `true` if the column COULD be null+// (no NotNull or PrimaryKey option); this returns `true` if it CANNOT.+func omniColumnIsNotNull(col *ast.ColumnDef) bool {+ if col == nil {+ return false+ }+ for _, c := range col.Constraints {+ if c == nil {+ continue+ }+ if c.Type == ast.ColConstrNotNull || c.Type == ast.ColConstrPrimaryKey {return true}}return false}++// omniColumnHasDefault reports whether the column has a DEFAULT clause.+// omni stores the default expression in `ColumnDef.DefaultValue` directly+// (not in Constraints[] like pingcap's column.Options[].Tp ==+// ColumnOptionDefaultValue). Direct field check — no fallible op.+func omniColumnHasDefault(col *ast.ColumnDef) bool {+ return col != nil && col.DefaultValue != nil+}diff --git a/backend/plugin/advisor/tidb/test/column_set_default_for_not_null.yaml b/backend/plugin/advisor/tidb/test/column_set_default_for_not_null.yamlindex e9c00294c6..d9cfe7df4b 100644--- a/backend/plugin/advisor/tidb/test/column_set_default_for_not_null.yaml+++ b/backend/plugin/advisor/tidb/test/column_set_default_for_not_null.yaml@@ -81,3 +81,25 @@endposition: null- statement: "\n\t\t\t\tCREATE TABLE book(uid int, id int);\n\t\t\t\tALTER TABLE book \n\t\t\t\t\tCHANGE COLUMN uid uid int NOT NULL DEFAULT 0,\n\t\t\t\t\tMODIFY COLUMN id int PRIMARY KEY DEFAULT 0,\n\t\t\t\t\tADD COLUMN name varchar(20) NOT NULL DEFAULT ''\n\t\t\t\t"changeType: 1+# omniNeedDefault contract pins (batch 7 migration). The following column+# kinds are NOT NULL but should NOT trigger the rule because they don't+# need an explicit DEFAULT clause:+# - BLOB / TEXT family (omni stores them under separate TypeName.Name+# strings — "BLOB", "TEXT", etc. — vs pingcap's unified TypeBlob enum;+# omniNeedDefault must include both name families)+# - JSON+# - AUTO_INCREMENT (auto-generated value)+# - GENERATED columns (computed from other columns)+# Without these pins, a future change that drops a name from the+# omniNeedDefault switch (or stops populating ColumnDef.AutoIncrement /+# ColumnDef.Generated) silently starts flagging these.+- statement: CREATE TABLE t(c TEXT NOT NULL)+ changeType: 1+- statement: CREATE TABLE t(c BLOB NOT NULL)+ changeType: 1+- statement: CREATE TABLE t(c JSON NOT NULL)+ changeType: 1+- statement: CREATE TABLE t(c INT NOT NULL AUTO_INCREMENT PRIMARY KEY)+ changeType: 1+- statement: CREATE TABLE t(a INT, c INT GENERATED ALWAYS AS (a + 1) STORED NOT NULL)+ changeType: 1diff --git a/backend/plugin/advisor/tidb/utils.go b/backend/plugin/advisor/tidb/utils.goindex d3904e39c2..84073b4a48 100644--- a/backend/plugin/advisor/tidb/utils.go+++ b/backend/plugin/advisor/tidb/utils.go@@ -300,15 +300,6 @@ func omniColumnHasComment(col *omniast.ColumnDef) bool {return false}-// getColumnCharset returns lowercased CHARACTER SET of a pingcap-AST-// column. Moved here from advisor_charset_allowlist.go during its omni-// migration; still consumed by advisor_column_disallow_set_charset.go-// (un-migrated). Delete when that advisor migrates.-// Tracked: https://linear.app/bytebase/issue/BYT-9412-func getColumnCharset(column *ast.ColumnDef) string {- return strings.ToLower(column.Tp.GetCharset())-}-// addColumnTargets returns the column definitions added by an ATAddColumn// cmd. omni populates either cmd.Columns (multi-column ADD COLUMN (...))// or cmd.Column (single ADD COLUMN); read both defensively.@@ -325,16 +316,43 @@ func addColumnTargets(cmd *omniast.AlterTableCmd) []*omniast.ColumnDef {return nil}-// canNull reports whether a pingcap-AST column may hold NULL (no NOT NULL-// or PRIMARY KEY option). Scoped to un-migrated advisors; delete when-// advisor_column_set_default_for_not_null.go migrates.-// Tracked: https://linear.app/bytebase/issue/BYT-9362-func canNull(column *ast.ColumnDef) bool {- for _, option := range column.Options {- if option.Tp == ast.ColumnOptionNotNull || option.Tp == ast.ColumnOptionPrimaryKey {+// omniNeedDefault reports whether a column needs a DEFAULT clause to be+// considered "well-formed" by the column_set_default_for_not_null and+// column_require_default rules. Returns false (default not needed) when:+// - column is auto-increment (auto-generated value)+// - column is generated (computed from other columns)+// - column is PRIMARY KEY (auto-incrementing-or-required by other means)+// - column type is BLOB/TEXT (omni stores TEXT as a separate name from+// BLOB; pingcap unified them under TypeBlob — both must be checked)+// - column type is JSON or geometry (no meaningful default)+//+// Mirror of pingcap-typed needDefault (still in utils.go for the+// un-migrated advisor_column_require_default.go consumer).+// When that advisor migrates, switch its calls here and delete needDefault.+// Tracked: https://linear.app/bytebase/issue/BYT-9414+func omniNeedDefault(col *omniast.ColumnDef) bool {+ if col == nil {+ return false+ }+ if col.AutoIncrement || col.Generated != nil {+ return false+ }+ for _, c := range col.Constraints {+ if c != nil && c.Type == omniast.ColConstrPrimaryKey {return false}}+ if col.TypeName == nil {+ return false+ }+ switch strings.ToUpper(col.TypeName.Name) {+ case "BLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB",+ "TEXT", "TINYTEXT", "MEDIUMTEXT", "LONGTEXT",+ "JSON",+ "GEOMETRY", "POINT", "LINESTRING", "POLYGON",+ "MULTIPOINT", "MULTILINESTRING", "MULTIPOLYGON", "GEOMETRYCOLLECTION":+ return false+ }return true}