Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(tidb): retain access tables for EXPLAIN ANALYZE (#21561)
p0ny4b7c26f8e416
diff --git a/backend/plugin/parser/tidb/query.go b/backend/plugin/parser/tidb/query.goindex 881f6290bb..1e6b63029a 100644--- a/backend/plugin/parser/tidb/query.go+++ b/backend/plugin/parser/tidb/query.go@@ -60,6 +60,66 @@ func ExtractMySQLTableList(in ast.Node, asName bool) []*ast.TableName {return extractTableList(in, input, asName)}+// Authorization needs every base table, including tables nested inside expressions+// and derived queries. The alias-oriented extractor below can collapse those tables.+type accessTableVisitor struct {+ tables []*ast.TableName+ resolveCTEs bool+ cteScopes []map[string]bool+ lockingTables map[*ast.TableName]bool+}++func (v *accessTableVisitor) Enter(node ast.Node) (ast.Node, bool) {+ if selectStmt, ok := node.(*ast.SelectStmt); ok && selectStmt.LockInfo != nil {+ if v.lockingTables == nil {+ v.lockingTables = make(map[*ast.TableName]bool)+ }+ // Locking clauses reference FROM aliases, whose physical sources are visited separately.+ for _, table := range selectStmt.LockInfo.Tables {+ v.lockingTables[table] = true+ }+ }+ if v.resolveCTEs {+ switch n := node.(type) {+ case *ast.SelectStmt, *ast.SetOprStmt, *ast.SetOprSelectList, *ast.UpdateStmt, *ast.DeleteStmt:+ v.cteScopes = append(v.cteScopes, make(map[string]bool))+ case *ast.CommonTableExpression:+ if n.IsRecursive {+ v.cteScopes[len(v.cteScopes)-1][n.Name.L] = true+ }+ default:+ }+ }+ if table, ok := node.(*ast.TableName); ok {+ if v.lockingTables[table] {+ return node, true+ }+ if v.resolveCTEs && table.Schema.O == "" {+ for _, scope := range v.cteScopes {+ if scope[table.Name.L] {+ return node, true+ }+ }+ }+ v.tables = append(v.tables, table)+ }+ return node, false+}++func (v *accessTableVisitor) Leave(node ast.Node) (ast.Node, bool) {+ if v.resolveCTEs {+ switch n := node.(type) {+ case *ast.SelectStmt, *ast.SetOprStmt, *ast.SetOprSelectList, *ast.UpdateStmt, *ast.DeleteStmt:+ v.cteScopes = v.cteScopes[:len(v.cteScopes)-1]+ case *ast.CommonTableExpression:+ // Non-recursive CTEs become visible only after their definition.+ v.cteScopes[len(v.cteScopes)-1][n.Name.L] = true+ default:+ }+ }+ return node, true+}+// -------------------------------------------- DO NOT TOUCH --------------------------------------------// extractTableList extracts all the TableNames from node.diff --git a/backend/plugin/parser/tidb/query_span_explain_test.go b/backend/plugin/parser/tidb/query_span_explain_test.gonew file mode 100644index 0000000000..ef528a6512--- /dev/null+++ b/backend/plugin/parser/tidb/query_span_explain_test.go@@ -0,0 +1,192 @@+package tidb++import (+ "context"+ "testing"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestExplainAnalyzeDigestRejected(t *testing.T) {+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{},+ base.Statement{Text: "EXPLAIN ANALYZE 'sqldigest'"}, "db", "", false)+ require.ErrorContains(t, err, "EXPLAIN ANALYZE by digest is not supported")+ require.Nil(t, span)+}++func TestExplainAnalyzeAccessTables(t *testing.T) {+ var metadata []*metadatapb.DatabaseSchemaMetadata+ for _, database := range []string{"db", "other"} {+ metadata = append(metadata, &metadatapb.DatabaseSchemaMetadata{+ Name: database,+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "",+ Tables: []*metadatapb.TableMetadata{+ {Name: "t", Columns: []*metadatapb.ColumnMetadata{{Name: "id"}}},+ {Name: "u", Columns: []*metadatapb.ColumnMetadata{{Name: "id"}}},+ },+ }},+ })+ }+ metadata = append(metadata, &metadatapb.DatabaseSchemaMetadata{+ Name: "information_schema", Schemas: []*metadatapb.SchemaMetadata{{Name: "", Tables: []*metadatapb.TableMetadata{{Name: "tables"}}}},+ })+ getter, lister := buildMockDatabaseMetadataGetter(metadata)+ for _, tc := range []struct {+ name string+ statement string+ queryType base.QueryType+ sources base.SourceColumnSet+ wantMixed bool+ }{+ {+ name: "count with predicate",+ statement: "EXPLAIN ANALYZE SELECT count(*) FROM t WHERE id = 839195265",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true},+ },+ {+ name: "join and cross database subquery",+ statement: "EXPLAIN ANALYZE SELECT a.id FROM t a JOIN u b ON a.id = b.id WHERE a.id IN (SELECT id FROM other.t)",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "db", Table: "u"}: true, {Database: "other", Table: "t"}: true},+ },+ {+ name: "derived join retains every table",+ statement: "EXPLAIN ANALYZE SELECT * FROM (SELECT t.id FROM t JOIN u ON t.id = u.id) x",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "db", Table: "u"}: true},+ },+ {+ name: "nested derived join across databases",+ statement: "EXPLAIN ANALYZE SELECT * FROM (SELECT * FROM (SELECT t.id FROM t JOIN other.t u ON t.id = u.id) x) y",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "other", Table: "t"}: true},+ },+ {+ name: "derived predicate subquery",+ statement: "EXPLAIN ANALYZE SELECT * FROM (SELECT id FROM t WHERE id IN (SELECT id FROM other.u)) x",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "other", Table: "u"}: true},+ },+ {+ name: "subqueries inside compound expressions",+ statement: "EXPLAIN ANALYZE SELECT COALESCE((SELECT max(id) FROM other.t), 0) FROM t WHERE id > 0 AND id IN (SELECT id FROM u)",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "db", Table: "u"}: true, {Database: "other", Table: "t"}: true},+ },+ {+ name: "qualified table",+ statement: "EXPLAIN ANALYZE SELECT count(*) FROM other.t",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "other", Table: "t"}: true},+ },+ {+ name: "uncached joined tables",+ statement: "EXPLAIN ANALYZE SELECT * FROM t JOIN missing ON TRUE JOIN other.missing2 ON TRUE",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "db", Table: "missing"}: true, {Database: "other", Table: "missing2"}: true},+ },+ {+ name: "uncached nested predicate",+ statement: "EXPLAIN ANALYZE SELECT * FROM t WHERE id IN (SELECT id FROM unsynced.missing)",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "unsynced", Table: "missing"}: true},+ },+ {+ name: "CTE references are not physical targets",+ statement: "EXPLAIN ANALYZE WITH c AS (SELECT * FROM missing) SELECT * FROM c JOIN t ON TRUE",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "missing"}: true, {Database: "db", Table: "t"}: true},+ },+ {+ name: "qualified table sharing CTE name",+ statement: "EXPLAIN ANALYZE WITH c AS (SELECT * FROM t) SELECT * FROM c JOIN other.c ON TRUE",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "other", Table: "c"}: true},+ },+ {+ name: "non recursive CTE reads same named physical table",+ statement: "EXPLAIN ANALYZE WITH t AS (SELECT * FROM t) SELECT * FROM t",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true},+ },+ {+ name: "recursive CTE reference",+ statement: "EXPLAIN ANALYZE WITH RECURSIVE c AS (SELECT id FROM missing UNION ALL SELECT id + 1 FROM c WHERE id < 5) SELECT * FROM c",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "missing"}: true},+ },+ {+ name: "nested CTE does not hide outer physical table",+ statement: "EXPLAIN ANALYZE SELECT * FROM (WITH missing AS (SELECT * FROM t) SELECT * FROM missing) x JOIN missing ON TRUE",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "db", Table: "missing"}: true},+ },+ {+ name: "plain explain permits mixed tables",+ statement: "EXPLAIN SELECT * FROM t JOIN information_schema.tables ON TRUE",+ queryType: base.Explain,+ sources: base.SourceColumnSet{},+ },+ {+ name: "analyze still rejects mixed tables",+ statement: "EXPLAIN ANALYZE SELECT * FROM t JOIN information_schema.tables ON TRUE",+ wantMixed: true,+ },+ {+ name: "locking aliases are not physical tables",+ statement: "EXPLAIN ANALYZE SELECT * FROM t AS a JOIN other.t AS b ON a.id = b.id FOR UPDATE OF a, b",+ queryType: base.Select,+ sources: base.SourceColumnSet{{Database: "db", Table: "t"}: true, {Database: "other", Table: "t"}: true},+ },+ {+ name: "plain explain",+ statement: "EXPLAIN SELECT * FROM t",+ queryType: base.Explain,+ sources: base.SourceColumnSet{},+ },+ {+ name: "analyzed write remains dml",+ statement: "EXPLAIN ANALYZE DELETE FROM t WHERE id = 1",+ queryType: base.DML,+ sources: base.SourceColumnSet{},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ }, base.Statement{Text: tc.statement}, "db", "", false)+ if tc.wantMixed {+ require.ErrorIs(t, err, base.MixUserSystemTablesError)+ return+ }+ require.NoError(t, err)+ require.Equal(t, tc.queryType, span.Type)+ require.Equal(t, tc.sources, span.SourceColumns)+ require.Empty(t, span.Results)+ })+ }+}++func TestRecursiveCTEShadowedInitialReference(t *testing.T) {+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Tables: []*metadatapb.TableMetadata{{Name: "t", Columns: []*metadatapb.ColumnMetadata{{Name: "id"}}}},+ }},+ }})+ statement := "WITH RECURSIVE c(id) AS (SELECT COALESCE((SELECT id FROM (WITH c AS (SELECT id FROM t) SELECT id FROM c) x), 0) UNION ALL SELECT id + 1 FROM c WHERE id < 10) SELECT id FROM c"+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ }, base.Statement{Text: statement}, "db", "", false)+ require.NoError(t, err)+ require.Nil(t, span.NotFoundError)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{{Database: "db", Table: "t"}: true}, span.SourceColumns)+}diff --git a/backend/plugin/parser/tidb/query_span_extractor.go b/backend/plugin/parser/tidb/query_span_extractor.goindex 42e694c67a..d501888b75 100644--- a/backend/plugin/parser/tidb/query_span_extractor.go+++ b/backend/plugin/parser/tidb/query_span_extractor.go@@ -63,7 +63,19 @@ func (q *querySpanExtractor) getQuerySpan(ctx context.Context, statement string)node := nodeList[0]- accessTables := q.getAccessTables(node)+ accessNode := node+ includeUncached := false+ if explain, ok := node.(*tidbast.ExplainStmt); ok {+ if !explain.Analyze {+ return &base.QuerySpan{Type: base.Explain, Results: []base.QuerySpanResult{}, SourceColumns: base.SourceColumnSet{}}, nil+ }+ if explain.Stmt == nil {+ return nil, errors.New("EXPLAIN ANALYZE by digest is not supported: referenced tables cannot be determined")+ }+ accessNode = explain.Stmt+ includeUncached = true+ }+ accessTables := q.getAccessTables(accessNode, includeUncached)allSystems, mixed := isMixedQuery(accessTables)if mixed {return nil, base.MixUserSystemTablesError@@ -191,10 +203,13 @@ var systemDatabases = map[string]bool{"metrics_schema": true,}-func (q *querySpanExtractor) getAccessTables(node tidbast.Node) base.SourceColumnSet {+func (q *querySpanExtractor) getAccessTables(node tidbast.Node, includeUncached bool) base.SourceColumnSet {accessesMap := make(base.SourceColumnSet)- tables := ExtractMySQLTableList(node, false /* asName */)- for _, table := range tables {+ // ANALYZE executes before metadata can be refreshed. Keep every physical+ // target for authorization, excluding only references to in-scope CTEs.+ v := &accessTableVisitor{resolveCTEs: includeUncached}+ node.Accept(v)+ for _, table := range v.tables {databaseName := table.Schema.Oif databaseName == "" {databaseName = q.defaultDatabase@@ -206,7 +221,7 @@ func (q *querySpanExtractor) getAccessTables(node tidbast.Node) base.SourceColum// We do this because we do not have too much time to implement the real behavior.// XXX(rebelice/zp): Can we pass more information here to make this function know the context and then// figure out whether the table is the table the query actually accesses- if !q.existsTableMetadata(databaseName, table.Name.O) {+ if !includeUncached && !q.existsTableMetadata(databaseName, table.Name.O) {continue}diff --git a/backend/plugin/parser/tidb/test-data/query_type.yaml b/backend/plugin/parser/tidb/test-data/query_type.yamlindex 45abc3c1ba..973b550490 100644--- a/backend/plugin/parser/tidb/test-data/query_type.yaml+++ b/backend/plugin/parser/tidb/test-data/query_type.yaml@@ -89,7 +89,12 @@querySpan:type: 1results: []- sourcecolumns: []+ sourcecolumns:+ - server: ""+ database: db+ schema: ""+ table: t+ column: ""predicatecolumns: []- description: Test Explainstatement: EXPLAIN SELECT t1.*, t2.* FROM t t1 JOIN (select * from t) t2 ON t1.a = t2.a;