Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(tidb): match names case-insensitively and resolve the nearest scope first in query span (#21494)
Vincent Huange36455b046a1
diff --git a/backend/plugin/parser/tidb/query_span_extractor.go b/backend/plugin/parser/tidb/query_span_extractor.goindex dc19acf5cf..0c1836b77f 100644--- a/backend/plugin/parser/tidb/query_span_extractor.go+++ b/backend/plugin/parser/tidb/query_span_extractor.go@@ -327,7 +327,7 @@ func (q *querySpanExtractor) extractRecursiveCTE(node *tidbast.CommonTableExpresfor i := previousCteOuterLength; i < len(q.ctes); i++ {cteTableSource := q.ctes[i]- if cteTableSource.Name == node.Name.O {+ if strings.EqualFold(cteTableSource.Name, node.Name.O) {// It means this recursive CTE will be hidden by the inner CTE with the same name.// In other words, this recursive CTE will be not references by itself sub-query.// So, we can build it as non-recursive CTE@@ -464,8 +464,8 @@ func (q *querySpanExtractor) extractSelect(node *tidbast.SelectStmt) (base.Tableresult.Columns = append(result.Columns, columns...)} else {for _, tableSource := range fromFieldList {- sameDatabase := (field.WildCard.Schema.O == tableSource.GetDatabaseName() || (field.WildCard.Schema.O == "" && tableSource.GetDatabaseName() == q.defaultDatabase))- sameTable := field.WildCard.Table.O == tableSource.GetTableName()+ sameDatabase := (strings.EqualFold(field.WildCard.Schema.O, tableSource.GetDatabaseName()) || (field.WildCard.Schema.O == "" && strings.EqualFold(tableSource.GetDatabaseName(), q.defaultDatabase)))+ sameTable := strings.EqualFold(field.WildCard.Table.O, tableSource.GetTableName())find := falseif sameDatabase && sameTable {result.Columns = append(result.Columns, tableSource.GetQuerySpanResult()...)@@ -629,13 +629,13 @@ func (q *querySpanExtractor) extractSourceColumnSetFromExpression(in tidbast.Expfunc (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName string) ([]base.QuerySpanResult, bool) {findInTableSource := func(tableSource base.TableSource) ([]base.QuerySpanResult, bool) {- if databaseName != "" && databaseName != tableSource.GetDatabaseName() {+ if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {return nil, false}- if databaseName == "" && tableSource.GetDatabaseName() != "" && tableSource.GetDatabaseName() != q.defaultDatabase {+ if databaseName == "" && tableSource.GetDatabaseName() != "" && !strings.EqualFold(tableSource.GetDatabaseName(), q.defaultDatabase) {return nil, false}- if tableName != "" && tableName != tableSource.GetTableName() {+ if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {return nil, false}// If the table name is empty, we should check if there are ambiguous fields,@@ -657,15 +657,17 @@ func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName st//// This query has two tables can be called `x1`, and the expression x1.a uses the closer x1 table.// This is the reason we loop the slice in reversed order.+ //+ // The statement's own FROM is searched first, as in getFieldColumnSource.- for i := len(q.outerTableSources) - 1; i >= 0; i-- {- tableSource := q.outerTableSources[i]+ for _, tableSource := range q.tableSourcesFrom {if sourceColumnSet, ok := findInTableSource(tableSource); ok {return sourceColumnSet, true}}- for _, tableSource := range q.tableSourcesFrom {+ for i := len(q.outerTableSources) - 1; i >= 0; i-- {+ tableSource := q.outerTableSources[i]if sourceColumnSet, ok := findInTableSource(tableSource); ok {return sourceColumnSet, true}@@ -707,15 +709,19 @@ func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, field//// This query has two tables can be called `x1`, and the expression x1.a uses the closer x1 table.// This is the reason we loop the slice in reversed order.+ //+ // The statement's own FROM is closer than any enclosing query, so it is+ // searched first. Otherwise a column the subquery reads is traced to a+ // same-named column of an outer table.- for i := len(q.outerTableSources) - 1; i >= 0; i-- {- tableSource := q.outerTableSources[i]+ for _, tableSource := range q.tableSourcesFrom {if sourceColumnSet, ok := findInTableSource(tableSource); ok {return sourceColumnSet, true}}- for _, tableSource := range q.tableSourcesFrom {+ for i := len(q.outerTableSources) - 1; i >= 0; i-- {+ tableSource := q.outerTableSources[i]if sourceColumnSet, ok := findInTableSource(tableSource); ok {return sourceColumnSet, true}@@ -831,7 +837,10 @@ func (q *querySpanExtractor) findTableSchema(databaseName string, tableName striif databaseName == "" {for i := len(q.ctes) - 1; i >= 0; i-- {cte := q.ctes[i]- if cte.Name == tableName {+ // TiDB fixes lower_case_table_names at 2, so `FROM c` reads CTE `C`+ // even when a base table `c` exists. Matching by case would trace+ // the CTE's columns to that table and apply its masking policy.+ if strings.EqualFold(cte.Name, tableName) {return cte, nil}}diff --git a/backend/plugin/parser/tidb/query_span_test.go b/backend/plugin/parser/tidb/query_span_test.goindex a493cb3bba..6de0a48c26 100644--- a/backend/plugin/parser/tidb/query_span_test.go+++ b/backend/plugin/parser/tidb/query_span_test.go@@ -232,6 +232,107 @@ func TestGetQuerySpanMissingTableUnionedWithAccessTables(t *testing.T) {a.True(foundUnknown, "not-found table must be unioned into SourceColumns so the pre-execute ACL check sees it")}+func TestGetQuerySpanNamesAreCaseInsensitive(t *testing.T) {+ metadata := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{+ {+ Name: "",+ Tables: []*metadatapb.TableMetadata{+ {Name: "c", Columns: []*metadatapb.ColumnMetadata{{Name: "s"}}},+ {Name: "sec", Columns: []*metadatapb.ColumnMetadata{{Name: "secret"}}},+ },+ },+ },+ }+ databaseMetadataGetter, databaseNamesLister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{metadata})++ // Base table `c` shares a name with CTE `C`; the CTE must win.+ for _, statement := range []string{+ "WITH C AS (SELECT secret AS s FROM sec) SELECT s FROM c",+ "WITH C AS (SELECT secret AS s FROM sec) SELECT * FROM c",+ "WITH C AS (SELECT secret AS s FROM sec) SELECT c.* FROM c",+ "WITH C AS (SELECT secret AS s FROM sec) SELECT C.s FROM c",+ "WITH RECURSIVE C AS (SELECT secret AS s, 1 AS n FROM sec UNION ALL SELECT s, n + 1 FROM c WHERE n < 2) SELECT s FROM c",+ "SELECT SEC.* FROM sec",+ "SELECT DB.sec.* FROM sec",+ "SELECT DB.SEC.secret FROM sec",+ } {+ t.Run(statement, func(t *testing.T) {+ span, err := GetQuerySpan(+ context.TODO(),+ base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: databaseMetadataGetter,+ ListDatabaseNamesFunc: databaseNamesLister,+ },+ base.Statement{Text: statement},+ "db",+ "",+ false,+ )+ require.NoError(t, err)+ require.NoError(t, span.NotFoundError)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{+ {Database: "db", Table: "sec", Column: "secret"}: true,+ }, span.Results[0].SourceColumns)+ })+ }+}++func TestGetQuerySpanResolvesNearestScopeFirst(t *testing.T) {+ metadata := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{+ {+ Name: "",+ Tables: []*metadatapb.TableMetadata{+ {Name: "pub", Columns: []*metadatapb.ColumnMetadata{{Name: "v"}}},+ {Name: "sec", Columns: []*metadatapb.ColumnMetadata{{Name: "v"}}},+ {Name: "d", Columns: []*metadatapb.ColumnMetadata{{Name: "k"}}},+ },+ },+ },+ }+ databaseMetadataGetter, databaseNamesLister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{metadata})++ tests := []struct {+ statement string+ wantTable string+ }{+ // The subquery's own FROM shadows a same-named column of the enclosing query.+ {"SELECT (SELECT v FROM sec LIMIT 1) AS r FROM pub", "sec"},+ {"SELECT (SELECT v FROM sec LIMIT 1) AS r FROM (SELECT 1 AS v) x", "sec"},+ {"SELECT (SELECT x.v FROM sec x LIMIT 1) AS r FROM pub x", "sec"},+ {"SELECT (SELECT t.* FROM sec AS t JOIN d AS x ON TRUE LIMIT 1) AS r FROM pub AS t", "sec"},+ {"SELECT (SELECT t.* FROM sec AS t JOIN d AS x ON TRUE LIMIT 1) AS r FROM pub AS T", "sec"},+ // A reference the subquery cannot satisfy still binds outward.+ {"SELECT (SELECT pub.v FROM sec LIMIT 1) AS r FROM pub", "pub"},+ {"SELECT (SELECT v FROM d LIMIT 1) AS r FROM pub", "pub"},+ }+ for _, tc := range tests {+ t.Run(tc.statement, func(t *testing.T) {+ span, err := GetQuerySpan(+ context.TODO(),+ base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: databaseMetadataGetter,+ ListDatabaseNamesFunc: databaseNamesLister,+ },+ base.Statement{Text: tc.statement},+ "db",+ "",+ false,+ )+ require.NoError(t, err)+ require.NoError(t, span.NotFoundError)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{+ {Database: "db", Table: tc.wantTable, Column: "v"}: true,+ }, span.Results[0].SourceColumns)+ })+ }+}+// TestGetQuerySpanCyclicViewReference pins that a cyclic view reference is// reported as an error rather than recursing until the stack overflows.// Ported from the MySQL guard (#20153).diff --git a/backend/plugin/parser/tidb/util.go b/backend/plugin/parser/tidb/util.goindex 2f283a6a84..72e7aada92 100644--- a/backend/plugin/parser/tidb/util.go+++ b/backend/plugin/parser/tidb/util.go@@ -1,6 +1,8 @@package tidbimport (+ "strings"+tidbast "github.com/pingcap/tidb/pkg/parser/ast")@@ -8,7 +10,7 @@ func splitInitialAndRecursivePart(node *tidbast.SetOprStmt, selfName string) ([]for i, selectStmt := range node.SelectList.Selects {tableList := ExtractMySQLTableList(selectStmt, false /* asName */)for _, table := range tableList {- if table.Schema.O == "" && table.Name.O == selfName {+ if table.Schema.O == "" && strings.EqualFold(table.Name.O, selfName) {return node.SelectList.Selects[:i], node.SelectList.Selects[i:]}}