Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: allow users access doris system tables (#17714)

rebelicedc1902357d3b
diff --git a/backend/plugin/parser/doris/query_span_extractor.go b/backend/plugin/parser/doris/query_span_extractor.go
index 1ee7069ad1..d7ad52d1a8 100644
--- a/backend/plugin/parser/doris/query_span_extractor.go
+++ b/backend/plugin/parser/doris/query_span_extractor.go
@@ -2,6 +2,7 @@ package doris
import (
"context"
+ "strings"
"github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/doris-parser"
@@ -10,19 +11,25 @@ import (
)
type querySpanExtractor struct {
+ ctx context.Context
defaultDatabase string
+ gCtx base.GetQuerySpanContext
// ctes tracks Common Table Expressions in the current scope
- ctes map[string]bool
+ ctes map[string]bool
+ ignoreCaseSensitive bool
}
-func newQuerySpanExtractor(database string, _ base.GetQuerySpanContext, _ bool) *querySpanExtractor {
+func newQuerySpanExtractor(database string, gCtx base.GetQuerySpanContext, ignoreCaseSensitive bool) *querySpanExtractor {
return &querySpanExtractor{
- defaultDatabase: database,
- ctes: make(map[string]bool),
+ defaultDatabase: database,
+ gCtx: gCtx,
+ ctes: make(map[string]bool),
+ ignoreCaseSensitive: ignoreCaseSensitive,
}
}
-func (q *querySpanExtractor) getQuerySpan(_ context.Context, statement string) (*base.QuerySpan, error) {
+func (q *querySpanExtractor) getQuerySpan(ctx context.Context, statement string) (*base.QuerySpan, error) {
+ q.ctx = ctx
parseResult, err := ParseDorisSQL(statement)
if err != nil {
return nil, err
@@ -35,10 +42,20 @@ func (q *querySpanExtractor) getQuerySpan(_ context.Context, statement string) (
}, nil
}
- accessTables := getAccessTables(q.defaultDatabase, parseResult, q.ctes)
+ accessTables := getAccessTables(q.defaultDatabase, parseResult, q.ctes, q.gCtx, q.ignoreCaseSensitive)
+
+ // We do not support simultaneous access to the system table and the user table
+ // because we do not synchronize the schema of the system table.
+ // This causes an error (NOT_FOUND) when using querySpanExtractor.findTableSchema.
+ // As a result, we exclude getting query span results for accessing only the system table.
+ allSystems, mixed := isMixedQuery(accessTables, q.ignoreCaseSensitive)
+ if mixed {
+ return nil, base.MixUserSystemTablesError
+ }
queryTypeListener := &queryTypeListener{
- result: base.QueryTypeUnknown,
+ allSystems: allSystems,
+ result: base.QueryTypeUnknown,
}
antlr.ParseTreeWalkerDefault.Walk(queryTypeListener, parseResult.Tree)
@@ -49,7 +66,7 @@ func (q *querySpanExtractor) getQuerySpan(_ context.Context, statement string) (
}, nil
}
-func getAccessTables(database string, parseResult *ParseResult, ctes map[string]bool) base.SourceColumnSet {
+func getAccessTables(database string, parseResult *ParseResult, ctes map[string]bool, gCtx base.GetQuerySpanContext, ignoreCaseSensitive bool) base.SourceColumnSet {
// First, extract CTEs from the query
cteListener := newCTEListener()
antlr.ParseTreeWalkerDefault.Walk(cteListener, parseResult.Tree)
@@ -59,7 +76,7 @@ func getAccessTables(database string, parseResult *ParseResult, ctes map[string]
ctes[cte] = true
}
- accessTableListener := newAccessTableListener(database, ctes)
+ accessTableListener := newAccessTableListener(database, ctes, gCtx, ignoreCaseSensitive)
antlr.ParseTreeWalkerDefault.Walk(accessTableListener, parseResult.Tree)
return accessTableListener.sourceColumnSet
@@ -68,16 +85,20 @@ func getAccessTables(database string, parseResult *ParseResult, ctes map[string]
type accessTableListener struct {
*parser.BaseDorisSQLListener
- defaultDatabase string
- sourceColumnSet base.SourceColumnSet
- ctes map[string]bool
+ defaultDatabase string
+ sourceColumnSet base.SourceColumnSet
+ ctes map[string]bool
+ gCtx base.GetQuerySpanContext
+ ignoreCaseSensitive bool
}
-func newAccessTableListener(database string, ctes map[string]bool) *accessTableListener {
+func newAccessTableListener(database string, ctes map[string]bool, gCtx base.GetQuerySpanContext, ignoreCaseSensitive bool) *accessTableListener {
return &accessTableListener{
- defaultDatabase: database,
- sourceColumnSet: base.SourceColumnSet{},
- ctes: ctes,
+ defaultDatabase: database,
+ sourceColumnSet: base.SourceColumnSet{},
+ ctes: ctes,
+ gCtx: gCtx,
+ ignoreCaseSensitive: ignoreCaseSensitive,
}
}
@@ -149,3 +170,39 @@ func (l *cteListener) extractCTEName(ctx *parser.CommonTableExpressionContext) {
l.ctes[cteName] = true
}
}
+
+// isMixedQuery checks whether the query accesses the user table and system table at the same time.
+// It returns whether all tables are system tables and whether there is a mixture.
+func isMixedQuery(m base.SourceColumnSet, ignoreCaseSensitive bool) (bool, bool) {
+ hasSystem, hasUser := false, false
+ for table := range m {
+ if isSystemResource(table, ignoreCaseSensitive) {
+ hasSystem = true
+ } else {
+ hasUser = true
+ }
+ }
+
+ if hasSystem && hasUser {
+ return false, true
+ }
+
+ return !hasUser && hasSystem, false
+}
+
+// systemDatabases contains Doris system databases
+// Reference: https://doris.apache.org/docs/3.x/admin-manual/system-tables/overview
+var systemDatabases = map[string]bool{
+ "information_schema": true,
+ "mysql": true,
+ "__internal_schema": true,
+ "_statistics_": true,
+}
+
+func isSystemResource(resource base.ColumnResource, ignoreCaseSensitive bool) bool {
+ database := resource.Database
+ if ignoreCaseSensitive {
+ database = strings.ToLower(database)
+ }
+ return systemDatabases[database]
+}
diff --git a/backend/plugin/parser/doris/query_span_extractor_system_table_test.go b/backend/plugin/parser/doris/query_span_extractor_system_table_test.go
new file mode 100644
index 0000000000..a0db052938
--- /dev/null
+++ b/backend/plugin/parser/doris/query_span_extractor_system_table_test.go
@@ -0,0 +1,95 @@
+package doris
+
+import (
+ "context"
+ "io"
+ "os"
+ "testing"
+
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+ "gopkg.in/yaml.v3"
+
+ "github.com/bytebase/bytebase/backend/common"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store/model"
+)
+
+func TestGetQuerySpan_SystemTable(t *testing.T) {
+ type testCase struct {
+ Description string `yaml:"description,omitempty"`
+ Statement string `yaml:"statement,omitempty"`
+ DefaultDatabase string `yaml:"defaultDatabase,omitempty"`
+ // Metadata is the protojson encoded storepb.DatabaseSchemaMetadata,
+ // if it's empty, we will use the defaultDatabaseMetadata.
+ Metadata string `yaml:"metadata,omitempty"`
+ QuerySpan *base.YamlQuerySpan `yaml:"querySpan,omitempty"`
+ }
+
+ const (
+ record = false
+ )
+
+ var (
+ testDataPaths = []string{"test-data/query_span_system_table.yaml"}
+ )
+
+ a := require.New(t)
+ for _, testDataPath := range testDataPaths {
+ yamlFile, err := os.Open(testDataPath)
+ a.NoError(err)
+
+ var testCases []testCase
+ byteValue, err := io.ReadAll(yamlFile)
+ a.NoError(err)
+ a.NoError(yamlFile.Close())
+ a.NoError(yaml.Unmarshal(byteValue, &testCases))
+
+ for i, tc := range testCases {
+ metadata := &storepb.DatabaseSchemaMetadata{}
+ a.NoError(common.ProtojsonUnmarshaler.Unmarshal([]byte(tc.Metadata), metadata))
+ databaseMetadataGetter, databaseNameLister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{metadata})
+ result, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{
+ GetDatabaseMetadataFunc: databaseMetadataGetter,
+ ListDatabaseNamesFunc: databaseNameLister,
+ }, tc.Statement, tc.DefaultDatabase, "", false)
+ a.NoErrorf(err, "statement: %s", tc.Statement)
+ resultYaml := result.ToYaml()
+ if record {
+ testCases[i].QuerySpan = resultYaml
+ } else {
+ a.Equalf(tc.QuerySpan, resultYaml, "statement: %s", tc.Statement)
+ }
+ }
+
+ if record {
+ byteValue, err := yaml.Marshal(testCases)
+ a.NoError(err)
+ err = os.WriteFile(testDataPath, byteValue, 0644)
+ a.NoError(err)
+ }
+ }
+}
+
+func buildMockDatabaseMetadataGetter(databaseMetadata []*storepb.DatabaseSchemaMetadata) (base.GetDatabaseMetadataFunc, base.ListDatabaseNamesFunc) {
+ return func(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {
+ m := make(map[string]*model.DatabaseMetadata)
+ for _, metadata := range databaseMetadata {
+ // Doris is case-insensitive for most objects
+ m[metadata.Name] = model.NewDatabaseMetadata(metadata, false /* isObjectCaseSensitive */, false /* isDetailCaseSensitive */)
+ }
+
+ if databaseMetadata, ok := m[databaseName]; ok {
+ return "", databaseMetadata, nil
+ }
+
+ return "", nil, errors.Errorf("database %q not found", databaseName)
+ }, func(_ context.Context, _ string) ([]string, error) {
+ var names []string
+ for _, metadata := range databaseMetadata {
+ names = append(names, metadata.Name)
+ }
+ return names, nil
+ }
+}
diff --git a/backend/plugin/parser/doris/query_type.go b/backend/plugin/parser/doris/query_type.go
index 0638f67e9e..a2a8c6a9cd 100644
--- a/backend/plugin/parser/doris/query_type.go
+++ b/backend/plugin/parser/doris/query_type.go
@@ -9,7 +9,8 @@ import (
type queryTypeListener struct {
*parser.BaseDorisSQLListener
- result base.QueryType
+ allSystems bool
+ result base.QueryType
}
func (l *queryTypeListener) EnterSingleStatement(ctx *parser.SingleStatementContext) {
@@ -24,7 +25,12 @@ func (l *queryTypeListener) EnterSingleStatement(ctx *parser.SingleStatementCont
switch {
case s.QueryStatement() != nil:
- l.result = base.Select
+ // If all tables are system tables, we should return SelectInfoSchema.
+ if l.allSystems {
+ l.result = base.SelectInfoSchema
+ } else {
+ l.result = base.Select
+ }
case s.InsertStatement() != nil, s.UpdateStatement() != nil, s.DeleteStatement() != nil:
l.result = base.DML
case s.ShowAlterStatement() != nil,
diff --git a/backend/plugin/parser/doris/test-data/query_span_system_table.yaml b/backend/plugin/parser/doris/test-data/query_span_system_table.yaml
new file mode 100644
index 0000000000..c62bbf3758
--- /dev/null
+++ b/backend/plugin/parser/doris/test-data/query_span_system_table.yaml
@@ -0,0 +1,107 @@
+- description: Query information_schema.tables
+ statement: SELECT * FROM information_schema.tables
+ defaultDatabase: testdb
+ metadata: '{}'
+ querySpan:
+ type: 3
+ results: []
+ sourcecolumns:
+ - server: ""
+ database: information_schema
+ schema: ""
+ table: tables
+ column: ""
+ predicatecolumns: []
+- description: Query information_schema.columns with WHERE clause
+ statement: SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'testdb'
+ defaultDatabase: testdb
+ metadata: '{}'
+ querySpan:
+ type: 3
+ results: []
+ sourcecolumns:
+ - server: ""
+ database: information_schema
+ schema: ""
+ table: columns
+ column: ""
+ predicatecolumns: []
+- description: Query _statistics_ database
+ statement: SELECT * FROM _statistics_.table_statistics
+ defaultDatabase: testdb
+ metadata: '{}'
+ querySpan:
+ type: 3
+ results: []
+ sourcecolumns:
+ - server: ""
+ database: _statistics_
+ schema: ""
+ table: table_statistics
+ column: ""
+ predicatecolumns: []
+- description: Query mysql database
+ statement: SELECT * FROM mysql.user
+ defaultDatabase: testdb
+ metadata: '{}'
+ querySpan:
+ type: 3
+ results: []
+ sourcecolumns:
+ - server: ""
+ database: mysql
+ schema: ""
+ table: user
+ column: ""
+ predicatecolumns: []
+- description: Query __internal_schema database
+ statement: SELECT * FROM __internal_schema.audit_log
+ defaultDatabase: testdb
+ metadata: '{}'
+ querySpan:
+ type: 3
+ results: []
+ sourcecolumns:
+ - server: ""
+ database: __internal_schema
+ schema: ""
+ table: audit_log
+ column: ""
+ predicatecolumns: []
+- description: Query user table
+ statement: SELECT * FROM users
+ defaultDatabase: testdb
+ metadata: |-
+ {
+ "name": "testdb",
+ "schemas": [
+ {
+ "name": "",
+ "tables": [
+ {
+ "name": "users",
+ "columns": [
+ {
+ "name": "id",
+ "position": 1
+ },
+ {
+ "name": "name",
+ "position": 2
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ querySpan:
+ type: 1
+ results: []
+ sourcecolumns:
+ - server: ""
+ database: testdb
+ schema: ""
+ table: users
+ column: ""
+ predicatecolumns: []