Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: allow users access doris system tables (#17714)
rebelicedc1902357d3b
diff --git a/backend/plugin/parser/doris/query_span_extractor.go b/backend/plugin/parser/doris/query_span_extractor.goindex 1ee7069ad1..d7ad52d1a8 100644--- a/backend/plugin/parser/doris/query_span_extractor.go+++ b/backend/plugin/parser/doris/query_span_extractor.go@@ -2,6 +2,7 @@ package dorisimport ("context"+ "strings""github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/doris-parser"@@ -10,19 +11,25 @@ import ()type querySpanExtractor struct {+ ctx context.ContextdefaultDatabase string+ gCtx base.GetQuerySpanContext// ctes tracks Common Table Expressions in the current scope- ctes map[string]bool+ ctes map[string]bool+ ignoreCaseSensitive bool}-func newQuerySpanExtractor(database string, _ base.GetQuerySpanContext, _ bool) *querySpanExtractor {+func newQuerySpanExtractor(database string, gCtx base.GetQuerySpanContext, ignoreCaseSensitive bool) *querySpanExtractor {return &querySpanExtractor{- defaultDatabase: database,- ctes: make(map[string]bool),+ defaultDatabase: database,+ gCtx: gCtx,+ ctes: make(map[string]bool),+ ignoreCaseSensitive: ignoreCaseSensitive,}}-func (q *querySpanExtractor) getQuerySpan(_ context.Context, statement string) (*base.QuerySpan, error) {+func (q *querySpanExtractor) getQuerySpan(ctx context.Context, statement string) (*base.QuerySpan, error) {+ q.ctx = ctxparseResult, err := ParseDorisSQL(statement)if err != nil {return nil, err@@ -35,10 +42,20 @@ func (q *querySpanExtractor) getQuerySpan(_ context.Context, statement string) (}, nil}- accessTables := getAccessTables(q.defaultDatabase, parseResult, q.ctes)+ accessTables := getAccessTables(q.defaultDatabase, parseResult, q.ctes, q.gCtx, q.ignoreCaseSensitive)++ // We do not support simultaneous access to the system table and the user table+ // because we do not synchronize the schema of the system table.+ // This causes an error (NOT_FOUND) when using querySpanExtractor.findTableSchema.+ // As a result, we exclude getting query span results for accessing only the system table.+ allSystems, mixed := isMixedQuery(accessTables, q.ignoreCaseSensitive)+ if mixed {+ return nil, base.MixUserSystemTablesError+ }queryTypeListener := &queryTypeListener{- result: base.QueryTypeUnknown,+ allSystems: allSystems,+ result: base.QueryTypeUnknown,}antlr.ParseTreeWalkerDefault.Walk(queryTypeListener, parseResult.Tree)@@ -49,7 +66,7 @@ func (q *querySpanExtractor) getQuerySpan(_ context.Context, statement string) (}, nil}-func getAccessTables(database string, parseResult *ParseResult, ctes map[string]bool) base.SourceColumnSet {+func getAccessTables(database string, parseResult *ParseResult, ctes map[string]bool, gCtx base.GetQuerySpanContext, ignoreCaseSensitive bool) base.SourceColumnSet {// First, extract CTEs from the querycteListener := newCTEListener()antlr.ParseTreeWalkerDefault.Walk(cteListener, parseResult.Tree)@@ -59,7 +76,7 @@ func getAccessTables(database string, parseResult *ParseResult, ctes map[string]ctes[cte] = true}- accessTableListener := newAccessTableListener(database, ctes)+ accessTableListener := newAccessTableListener(database, ctes, gCtx, ignoreCaseSensitive)antlr.ParseTreeWalkerDefault.Walk(accessTableListener, parseResult.Tree)return accessTableListener.sourceColumnSet@@ -68,16 +85,20 @@ func getAccessTables(database string, parseResult *ParseResult, ctes map[string]type accessTableListener struct {*parser.BaseDorisSQLListener- defaultDatabase string- sourceColumnSet base.SourceColumnSet- ctes map[string]bool+ defaultDatabase string+ sourceColumnSet base.SourceColumnSet+ ctes map[string]bool+ gCtx base.GetQuerySpanContext+ ignoreCaseSensitive bool}-func newAccessTableListener(database string, ctes map[string]bool) *accessTableListener {+func newAccessTableListener(database string, ctes map[string]bool, gCtx base.GetQuerySpanContext, ignoreCaseSensitive bool) *accessTableListener {return &accessTableListener{- defaultDatabase: database,- sourceColumnSet: base.SourceColumnSet{},- ctes: ctes,+ defaultDatabase: database,+ sourceColumnSet: base.SourceColumnSet{},+ ctes: ctes,+ gCtx: gCtx,+ ignoreCaseSensitive: ignoreCaseSensitive,}}@@ -149,3 +170,39 @@ func (l *cteListener) extractCTEName(ctx *parser.CommonTableExpressionContext) {l.ctes[cteName] = true}}++// isMixedQuery checks whether the query accesses the user table and system table at the same time.+// It returns whether all tables are system tables and whether there is a mixture.+func isMixedQuery(m base.SourceColumnSet, ignoreCaseSensitive bool) (bool, bool) {+ hasSystem, hasUser := false, false+ for table := range m {+ if isSystemResource(table, ignoreCaseSensitive) {+ hasSystem = true+ } else {+ hasUser = true+ }+ }++ if hasSystem && hasUser {+ return false, true+ }++ return !hasUser && hasSystem, false+}++// systemDatabases contains Doris system databases+// Reference: https://doris.apache.org/docs/3.x/admin-manual/system-tables/overview+var systemDatabases = map[string]bool{+ "information_schema": true,+ "mysql": true,+ "__internal_schema": true,+ "_statistics_": true,+}++func isSystemResource(resource base.ColumnResource, ignoreCaseSensitive bool) bool {+ database := resource.Database+ if ignoreCaseSensitive {+ database = strings.ToLower(database)+ }+ return systemDatabases[database]+}diff --git a/backend/plugin/parser/doris/query_span_extractor_system_table_test.go b/backend/plugin/parser/doris/query_span_extractor_system_table_test.gonew file mode 100644index 0000000000..a0db052938--- /dev/null+++ b/backend/plugin/parser/doris/query_span_extractor_system_table_test.go@@ -0,0 +1,95 @@+package doris++import (+ "context"+ "io"+ "os"+ "testing"++ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"+ "gopkg.in/yaml.v3"++ "github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model"+)++func TestGetQuerySpan_SystemTable(t *testing.T) {+ type testCase struct {+ Description string `yaml:"description,omitempty"`+ Statement string `yaml:"statement,omitempty"`+ DefaultDatabase string `yaml:"defaultDatabase,omitempty"`+ // Metadata is the protojson encoded storepb.DatabaseSchemaMetadata,+ // if it's empty, we will use the defaultDatabaseMetadata.+ Metadata string `yaml:"metadata,omitempty"`+ QuerySpan *base.YamlQuerySpan `yaml:"querySpan,omitempty"`+ }++ const (+ record = false+ )++ var (+ testDataPaths = []string{"test-data/query_span_system_table.yaml"}+ )++ a := require.New(t)+ for _, testDataPath := range testDataPaths {+ yamlFile, err := os.Open(testDataPath)+ a.NoError(err)++ var testCases []testCase+ byteValue, err := io.ReadAll(yamlFile)+ a.NoError(err)+ a.NoError(yamlFile.Close())+ a.NoError(yaml.Unmarshal(byteValue, &testCases))++ for i, tc := range testCases {+ metadata := &storepb.DatabaseSchemaMetadata{}+ a.NoError(common.ProtojsonUnmarshaler.Unmarshal([]byte(tc.Metadata), metadata))+ databaseMetadataGetter, databaseNameLister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{metadata})+ result, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: databaseMetadataGetter,+ ListDatabaseNamesFunc: databaseNameLister,+ }, tc.Statement, tc.DefaultDatabase, "", false)+ a.NoErrorf(err, "statement: %s", tc.Statement)+ resultYaml := result.ToYaml()+ if record {+ testCases[i].QuerySpan = resultYaml+ } else {+ a.Equalf(tc.QuerySpan, resultYaml, "statement: %s", tc.Statement)+ }+ }++ if record {+ byteValue, err := yaml.Marshal(testCases)+ a.NoError(err)+ err = os.WriteFile(testDataPath, byteValue, 0644)+ a.NoError(err)+ }+ }+}++func buildMockDatabaseMetadataGetter(databaseMetadata []*storepb.DatabaseSchemaMetadata) (base.GetDatabaseMetadataFunc, base.ListDatabaseNamesFunc) {+ return func(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {+ m := make(map[string]*model.DatabaseMetadata)+ for _, metadata := range databaseMetadata {+ // Doris is case-insensitive for most objects+ m[metadata.Name] = model.NewDatabaseMetadata(metadata, false /* isObjectCaseSensitive */, false /* isDetailCaseSensitive */)+ }++ if databaseMetadata, ok := m[databaseName]; ok {+ return "", databaseMetadata, nil+ }++ return "", nil, errors.Errorf("database %q not found", databaseName)+ }, func(_ context.Context, _ string) ([]string, error) {+ var names []string+ for _, metadata := range databaseMetadata {+ names = append(names, metadata.Name)+ }+ return names, nil+ }+}diff --git a/backend/plugin/parser/doris/query_type.go b/backend/plugin/parser/doris/query_type.goindex 0638f67e9e..a2a8c6a9cd 100644--- a/backend/plugin/parser/doris/query_type.go+++ b/backend/plugin/parser/doris/query_type.go@@ -9,7 +9,8 @@ import (type queryTypeListener struct {*parser.BaseDorisSQLListener- result base.QueryType+ allSystems bool+ result base.QueryType}func (l *queryTypeListener) EnterSingleStatement(ctx *parser.SingleStatementContext) {@@ -24,7 +25,12 @@ func (l *queryTypeListener) EnterSingleStatement(ctx *parser.SingleStatementContswitch {case s.QueryStatement() != nil:- l.result = base.Select+ // If all tables are system tables, we should return SelectInfoSchema.+ if l.allSystems {+ l.result = base.SelectInfoSchema+ } else {+ l.result = base.Select+ }case s.InsertStatement() != nil, s.UpdateStatement() != nil, s.DeleteStatement() != nil:l.result = base.DMLcase s.ShowAlterStatement() != nil,diff --git a/backend/plugin/parser/doris/test-data/query_span_system_table.yaml b/backend/plugin/parser/doris/test-data/query_span_system_table.yamlnew file mode 100644index 0000000000..c62bbf3758--- /dev/null+++ b/backend/plugin/parser/doris/test-data/query_span_system_table.yaml@@ -0,0 +1,107 @@+- description: Query information_schema.tables+ statement: SELECT * FROM information_schema.tables+ defaultDatabase: testdb+ metadata: '{}'+ querySpan:+ type: 3+ results: []+ sourcecolumns:+ - server: ""+ database: information_schema+ schema: ""+ table: tables+ column: ""+ predicatecolumns: []+- description: Query information_schema.columns with WHERE clause+ statement: SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'testdb'+ defaultDatabase: testdb+ metadata: '{}'+ querySpan:+ type: 3+ results: []+ sourcecolumns:+ - server: ""+ database: information_schema+ schema: ""+ table: columns+ column: ""+ predicatecolumns: []+- description: Query _statistics_ database+ statement: SELECT * FROM _statistics_.table_statistics+ defaultDatabase: testdb+ metadata: '{}'+ querySpan:+ type: 3+ results: []+ sourcecolumns:+ - server: ""+ database: _statistics_+ schema: ""+ table: table_statistics+ column: ""+ predicatecolumns: []+- description: Query mysql database+ statement: SELECT * FROM mysql.user+ defaultDatabase: testdb+ metadata: '{}'+ querySpan:+ type: 3+ results: []+ sourcecolumns:+ - server: ""+ database: mysql+ schema: ""+ table: user+ column: ""+ predicatecolumns: []+- description: Query __internal_schema database+ statement: SELECT * FROM __internal_schema.audit_log+ defaultDatabase: testdb+ metadata: '{}'+ querySpan:+ type: 3+ results: []+ sourcecolumns:+ - server: ""+ database: __internal_schema+ schema: ""+ table: audit_log+ column: ""+ predicatecolumns: []+- description: Query user table+ statement: SELECT * FROM users+ defaultDatabase: testdb+ metadata: |-+ {+ "name": "testdb",+ "schemas": [+ {+ "name": "",+ "tables": [+ {+ "name": "users",+ "columns": [+ {+ "name": "id",+ "position": 1+ },+ {+ "name": "name",+ "position": 2+ }+ ]+ }+ ]+ }+ ]+ }+ querySpan:+ type: 1+ results: []+ sourcecolumns:+ - server: ""+ database: testdb+ schema: ""+ table: users+ column: ""+ predicatecolumns: []