Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(tidb): resolve _tidb_rowid reads in query span (#21495)

Vincent Huangcec195c5f2fc
diff --git a/backend/plugin/parser/tidb/query_span_extractor.go b/backend/plugin/parser/tidb/query_span_extractor.go
index 0c1836b77f..42e694c67a 100644
--- a/backend/plugin/parser/tidb/query_span_extractor.go
+++ b/backend/plugin/parser/tidb/query_span_extractor.go
@@ -12,6 +12,11 @@ import (
"github.com/bytebase/bytebase/backend/store/model"
)
+// rowIDColumnName is TiDB's hidden row handle: readable on a table without a
+// clustered index, listed in no catalog, so never in synced metadata.
+// https://docs.pingcap.com/tidb/stable/tidb-rowid/
+const rowIDColumnName = "_tidb_rowid"
+
type querySpanExtractor struct {
ctx context.Context
defaultDatabase string
@@ -23,6 +28,10 @@ type querySpanExtractor struct {
outerTableSources []base.TableSource
tableSourcesFrom []base.TableSource
+ // baseTableSources marks the aliases and joins built only from base
+ // tables; base.PseudoTable drops that identity.
+ baseTableSources map[base.TableSource]bool
+
// viewResolutionStack tracks the views currently being expanded so a
// cyclic view reference (a view that transitively references itself) is
// reported as an error instead of recursing until the stack overflows.
@@ -35,6 +44,7 @@ func newQuerySpanExtractor(defaultDatabase string, gCtx base.GetQuerySpanContext
metaCache: make(map[string]*model.DatabaseMetadata),
lowerTableViewMap: make(map[string]map[string]bool),
gCtx: gCtx,
+ baseTableSources: make(map[base.TableSource]bool),
}
}
@@ -526,15 +536,19 @@ func (q *querySpanExtractor) extractSourceColumnSetFromExpression(in tidbast.Exp
case *tidbast.ColumnNameExpr:
database, table, column := node.Name.Schema.O, node.Name.Table.O, node.Name.Name.O
sources, ok := q.getFieldColumnSource(database, table, column)
- if !ok {
- return base.SourceColumnSet{}, &base.ResourceNotFoundError{
- Err: errors.New("cannot find the column ref"),
- Database: &database,
- Table: &table,
- Column: &column,
- }
+ if ok {
+ return sources, nil
+ }
+ // The row handle carries no column data, hence no lineage.
+ if q.isRowIDReference(database, table, column) {
+ return base.SourceColumnSet{}, nil
+ }
+ return base.SourceColumnSet{}, &base.ResourceNotFoundError{
+ Err: errors.New("cannot find the column ref"),
+ Database: &database,
+ Table: &table,
+ Column: &column,
}
- return sources, nil
case *tidbast.BinaryOperationExpr:
return q.extractSourceColumnSetFromExpressionList([]tidbast.ExprNode{node.L, node.R})
case *tidbast.UnaryOperationExpr:
@@ -569,6 +583,7 @@ func (q *querySpanExtractor) extractSourceColumnSetFromExpression(in tidbast.Exp
ctes: q.ctes,
outerTableSources: append(q.outerTableSources, q.tableSourcesFrom...),
tableSourcesFrom: []base.TableSource{},
+ baseTableSources: q.baseTableSources,
viewResolutionStack: cloneViewResolutionStack(q.viewResolutionStack),
}
tableSource, err := subqueryExtractor.extractTableSourceFromNode(node.Query)
@@ -658,7 +673,7 @@ func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName st
// This query has two tables can be called `x1`, and the expression x1.a uses the closer x1 table.
// This is the reason we loop the slice in reversed order.
//
- // The statement's own FROM is searched first, as in getFieldColumnSource.
+ // The statement's own FROM is searched first, as in candidateTableSources.
for _, tableSource := range q.tableSourcesFrom {
if sourceColumnSet, ok := findInTableSource(tableSource); ok {
@@ -677,23 +692,56 @@ func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName st
}
func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldName string) (base.SourceColumnSet, bool) {
- findInTableSource := func(tableSource base.TableSource) (base.SourceColumnSet, bool) {
- if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {
- return nil, false
- }
- if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {
- return nil, false
- }
- // If the table name is empty, we should check if there are ambiguous fields,
- // but we delegate this responsibility to the db-server, we do the fail-open strategy here.
-
- querySpanResult := tableSource.GetQuerySpanResult()
- for _, field := range querySpanResult {
+ // If the table name is empty, we should check if there are ambiguous fields,
+ // but we delegate this responsibility to the db-server, we do the fail-open strategy here.
+ for _, tableSource := range q.candidateTableSources(databaseName, tableName) {
+ for _, field := range tableSource.GetQuerySpanResult() {
if strings.EqualFold(field.Name, fieldName) {
return field.SourceColumns, true
}
}
- return nil, false
+ }
+ return base.SourceColumnSet{}, false
+}
+
+// isRowIDReference reports whether an unresolved column reference is the row
+// handle. Every table source it can bind to must be built from base tables,
+// where TiDB rejects this name for a real column (ERROR 1166). A view or
+// subquery can own one that cached metadata lacks, and only the not-found
+// error triggers the resync that finds it: on TiDB 8.5,
+// `SELECT _tidb_rowid FROM clustered_tbl, such_view` reads the view's column.
+func (q *querySpanExtractor) isRowIDReference(databaseName, tableName, fieldName string) bool {
+ if !strings.EqualFold(fieldName, rowIDColumnName) {
+ return false
+ }
+ candidates := q.candidateTableSources(databaseName, tableName)
+ for _, tableSource := range candidates {
+ if !q.isBaseTableSource(tableSource) {
+ return false
+ }
+ }
+ return len(candidates) > 0
+}
+
+func (q *querySpanExtractor) isBaseTableSource(tableSource base.TableSource) bool {
+ if _, ok := tableSource.(*base.PhysicalTable); ok {
+ return true
+ }
+ return q.baseTableSources[tableSource]
+}
+
+// candidateTableSources returns the table sources a column reference with the
+// given qualifiers can bind to, in resolution order.
+func (q *querySpanExtractor) candidateTableSources(databaseName, tableName string) []base.TableSource {
+ var candidates []base.TableSource
+ appendIfMatch := func(tableSource base.TableSource) {
+ if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {
+ return
+ }
+ if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {
+ return
+ }
+ candidates = append(candidates, tableSource)
}
// One sub-query may have multi-outer schemas and the multi-outer schemas can use the same name, such as:
@@ -715,19 +763,12 @@ func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, field
// same-named column of an outer table.
for _, tableSource := range q.tableSourcesFrom {
- if sourceColumnSet, ok := findInTableSource(tableSource); ok {
- return sourceColumnSet, true
- }
+ appendIfMatch(tableSource)
}
-
for i := len(q.outerTableSources) - 1; i >= 0; i-- {
- tableSource := q.outerTableSources[i]
- if sourceColumnSet, ok := findInTableSource(tableSource); ok {
- return sourceColumnSet, true
- }
+ appendIfMatch(q.outerTableSources[i])
}
-
- return base.SourceColumnSet{}, false
+ return candidates
}
func (q *querySpanExtractor) extractJoin(node *tidbast.Join) (base.TableSource, error) {
@@ -746,7 +787,11 @@ func (q *querySpanExtractor) extractJoin(node *tidbast.Join) (base.TableSource,
}
q.tableSourcesFrom = append(q.tableSourcesFrom, rightTableSource)
q.tableSourcesFrom = append(q.tableSourcesFrom, rightTableSource)
- return q.mergeJoinTableSource(node, leftTableSource, rightTableSource), nil
+ joined := q.mergeJoinTableSource(node, leftTableSource, rightTableSource)
+ if q.isBaseTableSource(leftTableSource) && q.isBaseTableSource(rightTableSource) {
+ q.baseTableSources[joined] = true
+ }
+ return joined, nil
}
func (*querySpanExtractor) mergeJoinTableSource(node *tidbast.Join, leftTableSource, rightTableSource base.TableSource) *base.PseudoTable {
@@ -819,7 +864,11 @@ func (q *querySpanExtractor) extractTableSource(node *tidbast.TableSource) (base
return nil, err
}
if node.AsName.O != "" {
- return base.NewPseudoTable(node.AsName.O, tableSource.GetQuerySpanResult()), nil
+ aliased := base.NewPseudoTable(node.AsName.O, tableSource.GetQuerySpanResult())
+ if q.isBaseTableSource(tableSource) {
+ q.baseTableSources[aliased] = true
+ }
+ return aliased, nil
}
return tableSource, nil
}
diff --git a/backend/plugin/parser/tidb/query_span_rowid_test.go b/backend/plugin/parser/tidb/query_span_rowid_test.go
new file mode 100644
index 0000000000..6c8fac7d0f
--- /dev/null
+++ b/backend/plugin/parser/tidb/query_span_rowid_test.go
@@ -0,0 +1,273 @@
+package tidb
+
+import (
+ "context"
+ "slices"
+ "testing"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// rowIDTestMetadata mirrors what schema sync stores: information_schema never
+// lists _tidb_rowid, so no table carries it. v_shadow is a view whose real
+// column is named _tidb_rowid, which TiDB allows for views but not base tables.
+func rowIDTestMetadata() []*metadatapb.DatabaseSchemaMetadata {
+ other := &metadatapb.DatabaseSchemaMetadata{
+ Name: "other",
+ Schemas: []*metadatapb.SchemaMetadata{
+ {Tables: []*metadatapb.TableMetadata{
+ {Name: "logs", Columns: []*metadatapb.ColumnMetadata{{Name: "phone"}}},
+ }},
+ },
+ }
+ app := &metadatapb.DatabaseSchemaMetadata{
+ Name: "app",
+ Schemas: []*metadatapb.SchemaMetadata{
+ {
+ Name: "",
+ Tables: []*metadatapb.TableMetadata{
+ {Name: "logs", Columns: []*metadatapb.ColumnMetadata{{Name: "site_code"}, {Name: "phone"}}},
+ {Name: "clustered_logs", Columns: []*metadatapb.ColumnMetadata{{Name: "id"}, {Name: "site_code"}}},
+ },
+ Views: []*metadatapb.ViewMetadata{
+ {Name: "v_plain", Definition: "SELECT site_code FROM logs"},
+ {Name: "v_shadow", Definition: "SELECT phone AS _tidb_rowid FROM logs"},
+ {Name: "v_rowid", Definition: "SELECT _tidb_rowid AS rid, phone FROM logs"},
+ },
+ },
+ },
+ }
+ return []*metadatapb.DatabaseSchemaMetadata{app, other}
+}
+
+func getRowIDTestSpan(t *testing.T, statement string) *base.QuerySpan {
+ t.Helper()
+ getter, lister := buildMockDatabaseMetadataGetter(rowIDTestMetadata())
+ span, err := GetQuerySpan(
+ context.TODO(),
+ base.GetQuerySpanContext{GetDatabaseMetadataFunc: getter, ListDatabaseNamesFunc: lister},
+ base.Statement{Text: statement},
+ "app",
+ "",
+ false,
+ )
+ require.NoError(t, err, statement)
+ require.NotNil(t, span, statement)
+ return span
+}
+
+type rowIDResult struct {
+ name string
+ // lineage lists the source columns as "database.table.column", sorted.
+ lineage []string
+}
+
+func flattenRowIDResults(span *base.QuerySpan) []rowIDResult {
+ var got []rowIDResult
+ for _, r := range span.Results {
+ lineage := []string{}
+ for c := range r.SourceColumns {
+ lineage = append(lineage, c.Database+"."+c.Table+"."+c.Column)
+ }
+ slices.Sort(lineage)
+ got = append(got, rowIDResult{name: r.Name, lineage: lineage})
+ }
+ return got
+}
+
+func TestGetQuerySpanTiDBRowIDResolves(t *testing.T) {
+ none := []string{}
+ phone := []string{"app.logs.phone"}
+ tests := []struct {
+ description string
+ statement string
+ want []rowIDResult
+ }{
+ {
+ description: "unqualified",
+ statement: "SELECT _tidb_rowid FROM logs",
+ want: []rowIDResult{{"_tidb_rowid", none}},
+ },
+ {
+ description: "table-qualified beside an ordinary column",
+ statement: "SELECT logs._tidb_rowid, phone FROM logs",
+ want: []rowIDResult{{"_tidb_rowid", none}, {"phone", phone}},
+ },
+ {
+ description: "database-qualified",
+ statement: "SELECT app.logs._tidb_rowid FROM app.logs",
+ want: []rowIDResult{{"_tidb_rowid", none}},
+ },
+ {
+ description: "through a table alias",
+ statement: "SELECT m._tidb_rowid FROM logs m",
+ want: []rowIDResult{{"_tidb_rowid", none}},
+ },
+ {
+ description: "backticked and upper case",
+ statement: "SELECT `_TIDB_ROWID` FROM logs",
+ want: []rowIDResult{{"_TIDB_ROWID", none}},
+ },
+ {
+ description: "inside aggregates beside an ordinary column",
+ statement: "SELECT MIN(_tidb_rowid) AS start_key, MAX(phone) AS p FROM logs",
+ want: []rowIDResult{{"start_key", none}, {"p", phone}},
+ },
+ {
+ description: "BYT-10230 customer query",
+ statement: `SELECT
+ FLOOR((t.row_num - 1) / 500) + 1 AS page_num,
+ MIN(t._tidb_rowid) AS start_key,
+ MAX(t._tidb_rowid) AS end_key,
+ COUNT(*) AS page_size
+FROM (
+ SELECT _tidb_rowid, ROW_NUMBER() OVER (ORDER BY _tidb_rowid) AS row_num
+ FROM app.logs WHERE site_code = "3129"
+) t
+GROUP BY page_num
+ORDER BY page_num`,
+ want: []rowIDResult{{"page_num", none}, {"start_key", none}, {"end_key", none}, {"page_size", none}},
+ },
+ {
+ description: "qualified in a join of base tables",
+ statement: "SELECT l._tidb_rowid, c.site_code FROM logs l JOIN clustered_logs c ON l.site_code = c.site_code",
+ want: []rowIDResult{{"_tidb_rowid", none}, {"site_code", []string{"app.clustered_logs.site_code"}}},
+ },
+ {
+ description: "unqualified over a join of base tables",
+ statement: "SELECT _tidb_rowid FROM logs l JOIN clustered_logs c ON l.site_code = c.site_code",
+ want: []rowIDResult{{"_tidb_rowid", none}},
+ },
+ {
+ description: "scalar subquery whose outer scope is a base table",
+ statement: "SELECT (SELECT _tidb_rowid FROM logs LIMIT 1) AS r FROM clustered_logs",
+ want: []rowIDResult{{"r", none}},
+ },
+ {
+ description: "subquery qualified to an aliased outer base table",
+ statement: "SELECT (SELECT n._tidb_rowid FROM clustered_logs LIMIT 1) AS r FROM logs n",
+ want: []rowIDResult{{"r", none}},
+ },
+ {
+ description: "subquery under an outer join of base tables",
+ statement: "SELECT (SELECT _tidb_rowid FROM logs LIMIT 1) AS r FROM logs l JOIN clustered_logs c ON l.site_code = c.site_code",
+ want: []rowIDResult{{"r", none}},
+ },
+ {
+ description: "same table name in two databases, qualified to the second",
+ statement: "SELECT other.logs._tidb_rowid, other.logs.phone FROM app.logs, other.logs",
+ want: []rowIDResult{{"_tidb_rowid", none}, {"phone", []string{"other.logs.phone"}}},
+ },
+ {
+ description: "view that projects the hidden column",
+ statement: "SELECT rid, phone FROM v_rowid",
+ want: []rowIDResult{{"rid", none}, {"phone", phone}},
+ },
+ {
+ description: "CTE that projects the hidden column",
+ statement: "WITH c AS (SELECT _tidb_rowid AS rid, phone FROM logs) SELECT rid, phone FROM c",
+ want: []rowIDResult{{"rid", none}, {"phone", phone}},
+ },
+ {
+ description: "qualified to the base table while a view shares the scope",
+ statement: "SELECT logs._tidb_rowid FROM logs, v_plain",
+ want: []rowIDResult{{"_tidb_rowid", none}},
+ },
+ {
+ description: "SELECT * still excludes the hidden column",
+ statement: "SELECT * FROM logs",
+ want: []rowIDResult{{"site_code", []string{"app.logs.site_code"}}, {"phone", phone}},
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.description, func(t *testing.T) {
+ span := getRowIDTestSpan(t, tc.statement)
+ require.NoError(t, span.NotFoundError)
+ require.Equal(t, tc.want, flattenRowIDResults(span))
+ })
+ }
+}
+
+func TestGetQuerySpanTiDBRowIDRealColumnWins(t *testing.T) {
+ phone := []string{"app.logs.phone"}
+ tests := []struct {
+ description string
+ statement string
+ want []rowIDResult
+ }{
+ {
+ description: "view column",
+ statement: "SELECT _tidb_rowid FROM v_shadow",
+ want: []rowIDResult{{"_tidb_rowid", phone}},
+ },
+ {
+ description: "derived-table column",
+ statement: "SELECT x._tidb_rowid FROM (SELECT phone AS _tidb_rowid FROM logs) x",
+ want: []rowIDResult{{"_tidb_rowid", phone}},
+ },
+ {
+ description: "CTE column, with the CTE named like a base table in another case",
+ statement: "WITH LOGS AS (SELECT phone AS _tidb_rowid FROM logs) SELECT _tidb_rowid FROM logs",
+ want: []rowIDResult{{"_tidb_rowid", phone}},
+ },
+ {
+ description: "qualified rowid falls outward to a derived column when the local table lacks it",
+ // TiDB binds x._tidb_rowid to the outer derived x because a clustered
+ // base table has no _tidb_rowid. The outer column is real and masked.
+ statement: "SELECT (SELECT x._tidb_rowid FROM clustered_logs x LIMIT 1) AS r FROM (SELECT phone AS _tidb_rowid FROM logs) x",
+ want: []rowIDResult{{"r", phone}},
+ },
+ {
+ description: "view column and hidden column side by side",
+ statement: "SELECT v_shadow._tidb_rowid, logs._tidb_rowid FROM logs, v_shadow",
+ want: []rowIDResult{{"_tidb_rowid", phone}, {"_tidb_rowid", []string{}}},
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.description, func(t *testing.T) {
+ span := getRowIDTestSpan(t, tc.statement)
+ require.NoError(t, span.NotFoundError)
+ require.Equal(t, tc.want, flattenRowIDResults(span))
+ })
+ }
+}
+
+func TestGetQuerySpanTiDBRowIDKeepsNotFound(t *testing.T) {
+ tests := []struct {
+ description string
+ statement string
+ wantColumn string
+ }{
+ {"view that does not project it", "SELECT _tidb_rowid FROM v_plain", "_tidb_rowid"},
+ {"aliased view", "SELECT v.`_tidb_rowid` FROM v_plain v", "_tidb_rowid"},
+ {"derived table that does not project it", "SELECT _tidb_rowid FROM (SELECT site_code FROM logs) x", "_tidb_rowid"},
+ {"qualified to such a derived table", "SELECT x._tidb_rowid FROM (SELECT site_code FROM logs) x", "_tidb_rowid"},
+ {"CTE that does not project it", "WITH c AS (SELECT site_code FROM logs) SELECT _tidb_rowid FROM c", "_tidb_rowid"},
+ {"unqualified with a view in the FROM scope", "SELECT _tidb_rowid FROM clustered_logs, v_plain", "_tidb_rowid"},
+ {"unqualified with a derived table in the FROM scope", "SELECT _tidb_rowid FROM logs JOIN (SELECT site_code FROM logs) x ON 1 = 1", "_tidb_rowid"},
+ {"qualifier bound only to an outer view", "SELECT (SELECT x._tidb_rowid FROM logs LIMIT 1) FROM v_plain x", "_tidb_rowid"},
+ {"qualifier shadowed in the subquery by a view", "SELECT (SELECT x._tidb_rowid FROM v_plain x LIMIT 1) FROM logs x", "_tidb_rowid"},
+ {"qualified rowid with the same alias on an outer view", "SELECT (SELECT x._tidb_rowid FROM logs x LIMIT 1) AS r FROM v_plain x", "_tidb_rowid"},
+ {"unqualified with a view in the outer scope", "SELECT (SELECT _tidb_rowid FROM clustered_logs LIMIT 1) FROM v_plain", "_tidb_rowid"},
+ {"no FROM clause", "SELECT _tidb_rowid", "_tidb_rowid"},
+ {"other unknown column on a base table", "SELECT nope FROM logs", "nope"},
+ {"name that only starts with the hidden column's", "SELECT _tidb_rowid_x FROM logs", "_tidb_rowid_x"},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.description, func(t *testing.T) {
+ span := getRowIDTestSpan(t, tc.statement)
+ require.Error(t, span.NotFoundError)
+ var resourceNotFound *base.ResourceNotFoundError
+ require.True(t, errors.As(span.NotFoundError, &resourceNotFound))
+ require.NotNil(t, resourceNotFound.Column)
+ require.Equal(t, tc.wantColumn, *resourceNotFound.Column)
+ })
+ }
+}