Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(tidb): resolve _tidb_rowid reads in query span (#21495)
Vincent Huangcec195c5f2fc
diff --git a/backend/plugin/parser/tidb/query_span_extractor.go b/backend/plugin/parser/tidb/query_span_extractor.goindex 0c1836b77f..42e694c67a 100644--- a/backend/plugin/parser/tidb/query_span_extractor.go+++ b/backend/plugin/parser/tidb/query_span_extractor.go@@ -12,6 +12,11 @@ import ("github.com/bytebase/bytebase/backend/store/model")+// rowIDColumnName is TiDB's hidden row handle: readable on a table without a+// clustered index, listed in no catalog, so never in synced metadata.+// https://docs.pingcap.com/tidb/stable/tidb-rowid/+const rowIDColumnName = "_tidb_rowid"+type querySpanExtractor struct {ctx context.ContextdefaultDatabase string@@ -23,6 +28,10 @@ type querySpanExtractor struct {outerTableSources []base.TableSourcetableSourcesFrom []base.TableSource+ // baseTableSources marks the aliases and joins built only from base+ // tables; base.PseudoTable drops that identity.+ baseTableSources map[base.TableSource]bool+// viewResolutionStack tracks the views currently being expanded so a// cyclic view reference (a view that transitively references itself) is// reported as an error instead of recursing until the stack overflows.@@ -35,6 +44,7 @@ func newQuerySpanExtractor(defaultDatabase string, gCtx base.GetQuerySpanContextmetaCache: make(map[string]*model.DatabaseMetadata),lowerTableViewMap: make(map[string]map[string]bool),gCtx: gCtx,+ baseTableSources: make(map[base.TableSource]bool),}}@@ -526,15 +536,19 @@ func (q *querySpanExtractor) extractSourceColumnSetFromExpression(in tidbast.Expcase *tidbast.ColumnNameExpr:database, table, column := node.Name.Schema.O, node.Name.Table.O, node.Name.Name.Osources, ok := q.getFieldColumnSource(database, table, column)- if !ok {- return base.SourceColumnSet{}, &base.ResourceNotFoundError{- Err: errors.New("cannot find the column ref"),- Database: &database,- Table: &table,- Column: &column,- }+ if ok {+ return sources, nil+ }+ // The row handle carries no column data, hence no lineage.+ if q.isRowIDReference(database, table, column) {+ return base.SourceColumnSet{}, nil+ }+ return base.SourceColumnSet{}, &base.ResourceNotFoundError{+ Err: errors.New("cannot find the column ref"),+ Database: &database,+ Table: &table,+ Column: &column,}- return sources, nilcase *tidbast.BinaryOperationExpr:return q.extractSourceColumnSetFromExpressionList([]tidbast.ExprNode{node.L, node.R})case *tidbast.UnaryOperationExpr:@@ -569,6 +583,7 @@ func (q *querySpanExtractor) extractSourceColumnSetFromExpression(in tidbast.Expctes: q.ctes,outerTableSources: append(q.outerTableSources, q.tableSourcesFrom...),tableSourcesFrom: []base.TableSource{},+ baseTableSources: q.baseTableSources,viewResolutionStack: cloneViewResolutionStack(q.viewResolutionStack),}tableSource, err := subqueryExtractor.extractTableSourceFromNode(node.Query)@@ -658,7 +673,7 @@ func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName st// This query has two tables can be called `x1`, and the expression x1.a uses the closer x1 table.// This is the reason we loop the slice in reversed order.//- // The statement's own FROM is searched first, as in getFieldColumnSource.+ // The statement's own FROM is searched first, as in candidateTableSources.for _, tableSource := range q.tableSourcesFrom {if sourceColumnSet, ok := findInTableSource(tableSource); ok {@@ -677,23 +692,56 @@ func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName st}func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldName string) (base.SourceColumnSet, bool) {- findInTableSource := func(tableSource base.TableSource) (base.SourceColumnSet, bool) {- if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {- return nil, false- }- if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {- return nil, false- }- // If the table name is empty, we should check if there are ambiguous fields,- // but we delegate this responsibility to the db-server, we do the fail-open strategy here.-- querySpanResult := tableSource.GetQuerySpanResult()- for _, field := range querySpanResult {+ // If the table name is empty, we should check if there are ambiguous fields,+ // but we delegate this responsibility to the db-server, we do the fail-open strategy here.+ for _, tableSource := range q.candidateTableSources(databaseName, tableName) {+ for _, field := range tableSource.GetQuerySpanResult() {if strings.EqualFold(field.Name, fieldName) {return field.SourceColumns, true}}- return nil, false+ }+ return base.SourceColumnSet{}, false+}++// isRowIDReference reports whether an unresolved column reference is the row+// handle. Every table source it can bind to must be built from base tables,+// where TiDB rejects this name for a real column (ERROR 1166). A view or+// subquery can own one that cached metadata lacks, and only the not-found+// error triggers the resync that finds it: on TiDB 8.5,+// `SELECT _tidb_rowid FROM clustered_tbl, such_view` reads the view's column.+func (q *querySpanExtractor) isRowIDReference(databaseName, tableName, fieldName string) bool {+ if !strings.EqualFold(fieldName, rowIDColumnName) {+ return false+ }+ candidates := q.candidateTableSources(databaseName, tableName)+ for _, tableSource := range candidates {+ if !q.isBaseTableSource(tableSource) {+ return false+ }+ }+ return len(candidates) > 0+}++func (q *querySpanExtractor) isBaseTableSource(tableSource base.TableSource) bool {+ if _, ok := tableSource.(*base.PhysicalTable); ok {+ return true+ }+ return q.baseTableSources[tableSource]+}++// candidateTableSources returns the table sources a column reference with the+// given qualifiers can bind to, in resolution order.+func (q *querySpanExtractor) candidateTableSources(databaseName, tableName string) []base.TableSource {+ var candidates []base.TableSource+ appendIfMatch := func(tableSource base.TableSource) {+ if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {+ return+ }+ if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {+ return+ }+ candidates = append(candidates, tableSource)}// One sub-query may have multi-outer schemas and the multi-outer schemas can use the same name, such as:@@ -715,19 +763,12 @@ func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, field// same-named column of an outer table.for _, tableSource := range q.tableSourcesFrom {- if sourceColumnSet, ok := findInTableSource(tableSource); ok {- return sourceColumnSet, true- }+ appendIfMatch(tableSource)}-for i := len(q.outerTableSources) - 1; i >= 0; i-- {- tableSource := q.outerTableSources[i]- if sourceColumnSet, ok := findInTableSource(tableSource); ok {- return sourceColumnSet, true- }+ appendIfMatch(q.outerTableSources[i])}-- return base.SourceColumnSet{}, false+ return candidates}func (q *querySpanExtractor) extractJoin(node *tidbast.Join) (base.TableSource, error) {@@ -746,7 +787,11 @@ func (q *querySpanExtractor) extractJoin(node *tidbast.Join) (base.TableSource,}q.tableSourcesFrom = append(q.tableSourcesFrom, rightTableSource)q.tableSourcesFrom = append(q.tableSourcesFrom, rightTableSource)- return q.mergeJoinTableSource(node, leftTableSource, rightTableSource), nil+ joined := q.mergeJoinTableSource(node, leftTableSource, rightTableSource)+ if q.isBaseTableSource(leftTableSource) && q.isBaseTableSource(rightTableSource) {+ q.baseTableSources[joined] = true+ }+ return joined, nil}func (*querySpanExtractor) mergeJoinTableSource(node *tidbast.Join, leftTableSource, rightTableSource base.TableSource) *base.PseudoTable {@@ -819,7 +864,11 @@ func (q *querySpanExtractor) extractTableSource(node *tidbast.TableSource) (basereturn nil, err}if node.AsName.O != "" {- return base.NewPseudoTable(node.AsName.O, tableSource.GetQuerySpanResult()), nil+ aliased := base.NewPseudoTable(node.AsName.O, tableSource.GetQuerySpanResult())+ if q.isBaseTableSource(tableSource) {+ q.baseTableSources[aliased] = true+ }+ return aliased, nil}return tableSource, nil}diff --git a/backend/plugin/parser/tidb/query_span_rowid_test.go b/backend/plugin/parser/tidb/query_span_rowid_test.gonew file mode 100644index 0000000000..6c8fac7d0f--- /dev/null+++ b/backend/plugin/parser/tidb/query_span_rowid_test.go@@ -0,0 +1,273 @@+package tidb++import (+ "context"+ "slices"+ "testing"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// rowIDTestMetadata mirrors what schema sync stores: information_schema never+// lists _tidb_rowid, so no table carries it. v_shadow is a view whose real+// column is named _tidb_rowid, which TiDB allows for views but not base tables.+func rowIDTestMetadata() []*metadatapb.DatabaseSchemaMetadata {+ other := &metadatapb.DatabaseSchemaMetadata{+ Name: "other",+ Schemas: []*metadatapb.SchemaMetadata{+ {Tables: []*metadatapb.TableMetadata{+ {Name: "logs", Columns: []*metadatapb.ColumnMetadata{{Name: "phone"}}},+ }},+ },+ }+ app := &metadatapb.DatabaseSchemaMetadata{+ Name: "app",+ Schemas: []*metadatapb.SchemaMetadata{+ {+ Name: "",+ Tables: []*metadatapb.TableMetadata{+ {Name: "logs", Columns: []*metadatapb.ColumnMetadata{{Name: "site_code"}, {Name: "phone"}}},+ {Name: "clustered_logs", Columns: []*metadatapb.ColumnMetadata{{Name: "id"}, {Name: "site_code"}}},+ },+ Views: []*metadatapb.ViewMetadata{+ {Name: "v_plain", Definition: "SELECT site_code FROM logs"},+ {Name: "v_shadow", Definition: "SELECT phone AS _tidb_rowid FROM logs"},+ {Name: "v_rowid", Definition: "SELECT _tidb_rowid AS rid, phone FROM logs"},+ },+ },+ },+ }+ return []*metadatapb.DatabaseSchemaMetadata{app, other}+}++func getRowIDTestSpan(t *testing.T, statement string) *base.QuerySpan {+ t.Helper()+ getter, lister := buildMockDatabaseMetadataGetter(rowIDTestMetadata())+ span, err := GetQuerySpan(+ context.TODO(),+ base.GetQuerySpanContext{GetDatabaseMetadataFunc: getter, ListDatabaseNamesFunc: lister},+ base.Statement{Text: statement},+ "app",+ "",+ false,+ )+ require.NoError(t, err, statement)+ require.NotNil(t, span, statement)+ return span+}++type rowIDResult struct {+ name string+ // lineage lists the source columns as "database.table.column", sorted.+ lineage []string+}++func flattenRowIDResults(span *base.QuerySpan) []rowIDResult {+ var got []rowIDResult+ for _, r := range span.Results {+ lineage := []string{}+ for c := range r.SourceColumns {+ lineage = append(lineage, c.Database+"."+c.Table+"."+c.Column)+ }+ slices.Sort(lineage)+ got = append(got, rowIDResult{name: r.Name, lineage: lineage})+ }+ return got+}++func TestGetQuerySpanTiDBRowIDResolves(t *testing.T) {+ none := []string{}+ phone := []string{"app.logs.phone"}+ tests := []struct {+ description string+ statement string+ want []rowIDResult+ }{+ {+ description: "unqualified",+ statement: "SELECT _tidb_rowid FROM logs",+ want: []rowIDResult{{"_tidb_rowid", none}},+ },+ {+ description: "table-qualified beside an ordinary column",+ statement: "SELECT logs._tidb_rowid, phone FROM logs",+ want: []rowIDResult{{"_tidb_rowid", none}, {"phone", phone}},+ },+ {+ description: "database-qualified",+ statement: "SELECT app.logs._tidb_rowid FROM app.logs",+ want: []rowIDResult{{"_tidb_rowid", none}},+ },+ {+ description: "through a table alias",+ statement: "SELECT m._tidb_rowid FROM logs m",+ want: []rowIDResult{{"_tidb_rowid", none}},+ },+ {+ description: "backticked and upper case",+ statement: "SELECT `_TIDB_ROWID` FROM logs",+ want: []rowIDResult{{"_TIDB_ROWID", none}},+ },+ {+ description: "inside aggregates beside an ordinary column",+ statement: "SELECT MIN(_tidb_rowid) AS start_key, MAX(phone) AS p FROM logs",+ want: []rowIDResult{{"start_key", none}, {"p", phone}},+ },+ {+ description: "BYT-10230 customer query",+ statement: `SELECT+ FLOOR((t.row_num - 1) / 500) + 1 AS page_num,+ MIN(t._tidb_rowid) AS start_key,+ MAX(t._tidb_rowid) AS end_key,+ COUNT(*) AS page_size+FROM (+ SELECT _tidb_rowid, ROW_NUMBER() OVER (ORDER BY _tidb_rowid) AS row_num+ FROM app.logs WHERE site_code = "3129"+) t+GROUP BY page_num+ORDER BY page_num`,+ want: []rowIDResult{{"page_num", none}, {"start_key", none}, {"end_key", none}, {"page_size", none}},+ },+ {+ description: "qualified in a join of base tables",+ statement: "SELECT l._tidb_rowid, c.site_code FROM logs l JOIN clustered_logs c ON l.site_code = c.site_code",+ want: []rowIDResult{{"_tidb_rowid", none}, {"site_code", []string{"app.clustered_logs.site_code"}}},+ },+ {+ description: "unqualified over a join of base tables",+ statement: "SELECT _tidb_rowid FROM logs l JOIN clustered_logs c ON l.site_code = c.site_code",+ want: []rowIDResult{{"_tidb_rowid", none}},+ },+ {+ description: "scalar subquery whose outer scope is a base table",+ statement: "SELECT (SELECT _tidb_rowid FROM logs LIMIT 1) AS r FROM clustered_logs",+ want: []rowIDResult{{"r", none}},+ },+ {+ description: "subquery qualified to an aliased outer base table",+ statement: "SELECT (SELECT n._tidb_rowid FROM clustered_logs LIMIT 1) AS r FROM logs n",+ want: []rowIDResult{{"r", none}},+ },+ {+ description: "subquery under an outer join of base tables",+ statement: "SELECT (SELECT _tidb_rowid FROM logs LIMIT 1) AS r FROM logs l JOIN clustered_logs c ON l.site_code = c.site_code",+ want: []rowIDResult{{"r", none}},+ },+ {+ description: "same table name in two databases, qualified to the second",+ statement: "SELECT other.logs._tidb_rowid, other.logs.phone FROM app.logs, other.logs",+ want: []rowIDResult{{"_tidb_rowid", none}, {"phone", []string{"other.logs.phone"}}},+ },+ {+ description: "view that projects the hidden column",+ statement: "SELECT rid, phone FROM v_rowid",+ want: []rowIDResult{{"rid", none}, {"phone", phone}},+ },+ {+ description: "CTE that projects the hidden column",+ statement: "WITH c AS (SELECT _tidb_rowid AS rid, phone FROM logs) SELECT rid, phone FROM c",+ want: []rowIDResult{{"rid", none}, {"phone", phone}},+ },+ {+ description: "qualified to the base table while a view shares the scope",+ statement: "SELECT logs._tidb_rowid FROM logs, v_plain",+ want: []rowIDResult{{"_tidb_rowid", none}},+ },+ {+ description: "SELECT * still excludes the hidden column",+ statement: "SELECT * FROM logs",+ want: []rowIDResult{{"site_code", []string{"app.logs.site_code"}}, {"phone", phone}},+ },+ }++ for _, tc := range tests {+ t.Run(tc.description, func(t *testing.T) {+ span := getRowIDTestSpan(t, tc.statement)+ require.NoError(t, span.NotFoundError)+ require.Equal(t, tc.want, flattenRowIDResults(span))+ })+ }+}++func TestGetQuerySpanTiDBRowIDRealColumnWins(t *testing.T) {+ phone := []string{"app.logs.phone"}+ tests := []struct {+ description string+ statement string+ want []rowIDResult+ }{+ {+ description: "view column",+ statement: "SELECT _tidb_rowid FROM v_shadow",+ want: []rowIDResult{{"_tidb_rowid", phone}},+ },+ {+ description: "derived-table column",+ statement: "SELECT x._tidb_rowid FROM (SELECT phone AS _tidb_rowid FROM logs) x",+ want: []rowIDResult{{"_tidb_rowid", phone}},+ },+ {+ description: "CTE column, with the CTE named like a base table in another case",+ statement: "WITH LOGS AS (SELECT phone AS _tidb_rowid FROM logs) SELECT _tidb_rowid FROM logs",+ want: []rowIDResult{{"_tidb_rowid", phone}},+ },+ {+ description: "qualified rowid falls outward to a derived column when the local table lacks it",+ // TiDB binds x._tidb_rowid to the outer derived x because a clustered+ // base table has no _tidb_rowid. The outer column is real and masked.+ statement: "SELECT (SELECT x._tidb_rowid FROM clustered_logs x LIMIT 1) AS r FROM (SELECT phone AS _tidb_rowid FROM logs) x",+ want: []rowIDResult{{"r", phone}},+ },+ {+ description: "view column and hidden column side by side",+ statement: "SELECT v_shadow._tidb_rowid, logs._tidb_rowid FROM logs, v_shadow",+ want: []rowIDResult{{"_tidb_rowid", phone}, {"_tidb_rowid", []string{}}},+ },+ }++ for _, tc := range tests {+ t.Run(tc.description, func(t *testing.T) {+ span := getRowIDTestSpan(t, tc.statement)+ require.NoError(t, span.NotFoundError)+ require.Equal(t, tc.want, flattenRowIDResults(span))+ })+ }+}++func TestGetQuerySpanTiDBRowIDKeepsNotFound(t *testing.T) {+ tests := []struct {+ description string+ statement string+ wantColumn string+ }{+ {"view that does not project it", "SELECT _tidb_rowid FROM v_plain", "_tidb_rowid"},+ {"aliased view", "SELECT v.`_tidb_rowid` FROM v_plain v", "_tidb_rowid"},+ {"derived table that does not project it", "SELECT _tidb_rowid FROM (SELECT site_code FROM logs) x", "_tidb_rowid"},+ {"qualified to such a derived table", "SELECT x._tidb_rowid FROM (SELECT site_code FROM logs) x", "_tidb_rowid"},+ {"CTE that does not project it", "WITH c AS (SELECT site_code FROM logs) SELECT _tidb_rowid FROM c", "_tidb_rowid"},+ {"unqualified with a view in the FROM scope", "SELECT _tidb_rowid FROM clustered_logs, v_plain", "_tidb_rowid"},+ {"unqualified with a derived table in the FROM scope", "SELECT _tidb_rowid FROM logs JOIN (SELECT site_code FROM logs) x ON 1 = 1", "_tidb_rowid"},+ {"qualifier bound only to an outer view", "SELECT (SELECT x._tidb_rowid FROM logs LIMIT 1) FROM v_plain x", "_tidb_rowid"},+ {"qualifier shadowed in the subquery by a view", "SELECT (SELECT x._tidb_rowid FROM v_plain x LIMIT 1) FROM logs x", "_tidb_rowid"},+ {"qualified rowid with the same alias on an outer view", "SELECT (SELECT x._tidb_rowid FROM logs x LIMIT 1) AS r FROM v_plain x", "_tidb_rowid"},+ {"unqualified with a view in the outer scope", "SELECT (SELECT _tidb_rowid FROM clustered_logs LIMIT 1) FROM v_plain", "_tidb_rowid"},+ {"no FROM clause", "SELECT _tidb_rowid", "_tidb_rowid"},+ {"other unknown column on a base table", "SELECT nope FROM logs", "nope"},+ {"name that only starts with the hidden column's", "SELECT _tidb_rowid_x FROM logs", "_tidb_rowid_x"},+ }++ for _, tc := range tests {+ t.Run(tc.description, func(t *testing.T) {+ span := getRowIDTestSpan(t, tc.statement)+ require.Error(t, span.NotFoundError)+ var resourceNotFound *base.ResourceNotFoundError+ require.True(t, errors.As(span.NotFoundError, &resourceNotFound))+ require.NotNil(t, resourceNotFound.Column)+ require.Equal(t, tc.wantColumn, *resourceNotFound.Column)+ })+ }+}