Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: choose admin data source for automatic DML queries (#20390)
rebelice996309b051f2
diff --git a/backend/api/v1/access_grant_service.go b/backend/api/v1/access_grant_service.goindex 9db2564652..02f1e50fc0 100644--- a/backend/api/v1/access_grant_service.go+++ b/backend/api/v1/access_grant_service.go@@ -17,7 +17,6 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/generated-go/v1/v1connect"- parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store")@@ -164,7 +163,7 @@ func (s *AccessGrantService) CreateAccessGrant(ctx context.Context, request *conif instance == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("instance %q not found", instanceID))}- if ok, _, err := parserbase.ValidateSQLForEditor(instance.Metadata.GetEngine(), ag.Query); err != nil {+ if ok, err := isReadOnlyStatementForAccessGrant(ctx, instance.Metadata.GetEngine(), ag.Query); err != nil {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Wrapf(err, "invalid query"))} else if !ok {return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("only read-only statements are allowed in access grants"))diff --git a/backend/api/v1/access_grant_service_test.go b/backend/api/v1/access_grant_service_test.gonew file mode 100644index 0000000000..74ce4956ad--- /dev/null+++ b/backend/api/v1/access_grant_service_test.go@@ -0,0 +1,80 @@+package v1++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++func TestIsReadOnlyStatementForAccessGrantRejectsDocumentEngineWriteStatements(t *testing.T) {+ tests := []struct {+ name string+ engine storepb.Engine+ statement string+ }{+ {+ name: "MongoDB DML",+ engine: storepb.Engine_MONGODB,+ statement: `db.users.insertOne({name: "Bytebase"})`,+ },+ {+ name: "MongoDB DDL",+ engine: storepb.Engine_MONGODB,+ statement: `db.createCollection("users")`,+ },+ {+ name: "Elasticsearch DML",+ engine: storepb.Engine_ELASTICSEARCH,+ statement: "POST /users/_doc\n{\"name\":\"Bytebase\"}",+ },+ {+ name: "Elasticsearch DDL",+ engine: storepb.Engine_ELASTICSEARCH,+ statement: "PUT /users",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ readOnly, err := isReadOnlyStatementForAccessGrant(context.Background(), tc.engine, tc.statement)+ require.NoError(t, err)+ require.False(t, readOnly)+ })+ }+}++func TestIsReadOnlyStatementForAccessGrantAllowsDocumentEngineReadStatements(t *testing.T) {+ tests := []struct {+ name string+ engine storepb.Engine+ statement string+ }{+ {+ name: "MongoDB read",+ engine: storepb.Engine_MONGODB,+ statement: `db.users.find({})`,+ },+ {+ name: "Elasticsearch read",+ engine: storepb.Engine_ELASTICSEARCH,+ statement: "GET /users/_search",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ readOnly, err := isReadOnlyStatementForAccessGrant(context.Background(), tc.engine, tc.statement)+ require.NoError(t, err)+ require.True(t, readOnly)+ })+ }+}++func TestIsReadOnlyStatementForAccessGrantRejectsDocumentEngineInvalidStatements(t *testing.T) {+ readOnly, err := isReadOnlyStatementForAccessGrant(context.Background(), storepb.Engine_ELASTICSEARCH, `db.users.find({})`)+ require.Error(t, err)+ require.False(t, readOnly)+}diff --git a/backend/api/v1/query_statement_classification.go b/backend/api/v1/query_statement_classification.gonew file mode 100644index 0000000000..0032919080--- /dev/null+++ b/backend/api/v1/query_statement_classification.go@@ -0,0 +1,59 @@+package v1++import (+ "context"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func isReadOnlyStatementForAccessGrant(ctx context.Context, engine storepb.Engine, statement string) (bool, error) {+ readOnly, _, err := parserbase.ValidateSQLForEditor(engine, statement)+ if err != nil {+ return false, err+ }+ if !readOnly {+ return false, nil+ }++ if !shouldClassifyStatementByQuerySpan(engine) {+ return true, nil+ }+ spans, err := getQuerySpansForStatement(ctx, engine, statement)+ if err != nil {+ return false, err+ }+ if len(spans) == 0 {+ return false, nil+ }+ for _, span := range spans {+ if span == nil {+ return false, nil+ }+ switch span.Type {+ case parserbase.Select, parserbase.SelectInfoSchema, parserbase.Explain:+ case parserbase.QueryTypeUnknown, parserbase.DDL, parserbase.DML:+ return false, nil+ default:+ return false, nil+ }+ }+ return true, nil+}++func getQuerySpansForStatement(ctx context.Context, engine storepb.Engine, statement string) ([]*parserbase.QuerySpan, error) {+ statements, err := parserbase.SplitMultiSQL(engine, statement)+ if err != nil {+ statements = []parserbase.Statement{{Text: statement}}+ }+ return parserbase.GetQuerySpan(ctx, parserbase.GetQuerySpanContext{}, engine, statements, "", "", false)+}++func shouldClassifyStatementByQuerySpan(engine storepb.Engine) bool {+ switch engine {+ case storepb.Engine_MONGODB, storepb.Engine_ELASTICSEARCH:+ return true+ default:+ return false+ }+}diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex c3b0a7fd8d..07caed4d5d 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -165,7 +165,7 @@ func (s *SQLService) AdminExecute(ctx context.Context, stream *connect.BidiStrea// preCheckAccess finds and returns the best matching active access grant for the query.// It lists access grants filtered by project, creator, status, statement, target database,// and expiry, then prefers the grant with unmask=true if available.-func (s *SQLService) preCheckAccess(ctx context.Context, request *v1pb.QueryRequest, database *store.DatabaseMessage) *store.AccessGrantMessage {+func (s *SQLService) preCheckAccess(ctx context.Context, request *v1pb.QueryRequest, instance *store.InstanceMessage, database *store.DatabaseMessage) *store.AccessGrantMessage {project, err := s.store.GetProject(ctx, &store.FindProjectMessage{Workspace: common.GetWorkspaceIDFromContext(ctx),ResourceID: &database.ProjectID,@@ -217,6 +217,15 @@ func (s *SQLService) preCheckAccess(ctx context.Context, request *v1pb.QueryRequif len(grants) == 0 {return nil}+ readOnly, err := isReadOnlyStatementForAccessGrant(ctx, instance.Metadata.GetEngine(), request.Statement)+ if err != nil {+ slog.Warn("failed to validate access grant query", log.BBError(err))+ return nil+ }+ if !readOnly {+ slog.Warn("skip access grant for non-read-only query", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName))+ return nil+ }// Pick the best grant (prefer unmask=true).for _, grant := range grants {if grant.Payload != nil && grant.Payload.Unmask {@@ -234,7 +243,7 @@ func (s *SQLService) Query(ctx context.Context, req *connect.Request[v1pb.QueryRreturn nil, err}- accessGrant := s.preCheckAccess(ctx, request, database)+ accessGrant := s.preCheckAccess(ctx, request, instance, database)statement := request.Statement// In Redshift datashare, Rewrite query used for parser.@@ -250,7 +259,14 @@ func (s *SQLService) Query(ctx context.Context, req *connect.Request[v1pb.QueryR}}- resolvedDataSourceID, err := resolveDataSourceID(instance, request.DataSourceId)+ queryDataPolicy := getEffectiveQueryDataPolicy(+ ctx,+ s.store,+ s.licenseService,+ 0,+ database.ProjectID,+ )+ resolvedDataSourceID, err := resolveDataSourceID(ctx, instance, request.DataSourceId, statement, queryDataPolicy.AllowAdminDataSource)if err != nil {return nil, err}@@ -878,7 +894,7 @@ func (s *SQLService) Export(ctx context.Context, req *connect.Request[v1pb.Expor}}- resolvedDataSourceID, err := resolveDataSourceID(instance, request.DataSourceId)+ resolvedDataSourceID, err := resolveDataSourceID(ctx, instance, request.DataSourceId, statement, false)if err != nil {return nil, err}@@ -1785,7 +1801,7 @@ func (*SQLService) DiffMetadata(_ context.Context, req *connect.Request[v1pb.Dif}), nil}-func resolveDataSourceID(instance *store.InstanceMessage, dataSourceID string) (string, error) {+func resolveDataSourceID(ctx context.Context, instance *store.InstanceMessage, dataSourceID string, statement string, allowAdminDataSource bool) (string, error) {if dataSourceID != "" {return dataSourceID, nil}@@ -1804,6 +1820,10 @@ func resolveDataSourceID(instance *store.InstanceMessage, dataSourceID string) (}}+ if allowAdminDataSource && adminDataSourceID != "" && requiresAdminDataSource(ctx, instance.Metadata.GetEngine(), statement) {+ return adminDataSourceID, nil+ }+switch {case readOnlyCount == 1:return readOnlyDataSourceID, nil@@ -1816,6 +1836,33 @@ func resolveDataSourceID(instance *store.InstanceMessage, dataSourceID string) (}}+func requiresAdminDataSource(ctx context.Context, engine storepb.Engine, statement string) bool {+ readOnly, _, err := parserbase.ValidateSQLForEditor(engine, statement)+ if err != nil {+ return false+ }+ if !readOnly {+ return true+ }++ if !shouldClassifyStatementByQuerySpan(engine) {+ return false+ }+ spans, err := getQuerySpansForStatement(ctx, engine, statement)+ if err != nil {+ return false+ }+ for _, span := range spans {+ if span == nil {+ continue+ }+ if span.Type == parserbase.DDL || span.Type == parserbase.DML {+ return true+ }+ }+ return false+}+func checkAndGetDataSourceQueriable(ctx context.Context,storeInstance *store.Store,diff --git a/backend/api/v1/sql_service_test.go b/backend/api/v1/sql_service_test.gonew file mode 100644index 0000000000..ab79c7f4e8--- /dev/null+++ b/backend/api/v1/sql_service_test.go@@ -0,0 +1,143 @@+package v1++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/store"+)++func TestResolveDataSourceIDUsesAdminForNonReadOnlyAutomaticQueryWhenAllowed(t *testing.T) {+ instance := &store.InstanceMessage{+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_MYSQL,+ DataSources: []*storepb.DataSource{+ {Id: "admin", Type: storepb.DataSourceType_ADMIN},+ {Id: "readonly", Type: storepb.DataSourceType_READ_ONLY},+ },+ },+ }++ got, err := resolveDataSourceID(context.Background(), instance, "", "INSERT INTO books VALUES (1, 'Bytebase');", true)+ require.NoError(t, err)+ require.Equal(t, "admin", got)+}++func TestResolveDataSourceIDKeepsReadOnlyForReadOnlyAutomaticQueryWhenAllowed(t *testing.T) {+ instance := &store.InstanceMessage{+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_MYSQL,+ DataSources: []*storepb.DataSource{+ {Id: "admin", Type: storepb.DataSourceType_ADMIN},+ {Id: "readonly", Type: storepb.DataSourceType_READ_ONLY},+ },+ },+ }++ got, err := resolveDataSourceID(context.Background(), instance, "", "SELECT * FROM books;", true)+ require.NoError(t, err)+ require.Equal(t, "readonly", got)+}++func TestResolveDataSourceIDKeepsReadOnlyForNonReadOnlyAutomaticQueryWhenAdminDisallowed(t *testing.T) {+ instance := &store.InstanceMessage{+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_MYSQL,+ DataSources: []*storepb.DataSource{+ {Id: "admin", Type: storepb.DataSourceType_ADMIN},+ {Id: "readonly", Type: storepb.DataSourceType_READ_ONLY},+ },+ },+ }++ got, err := resolveDataSourceID(context.Background(), instance, "", "INSERT INTO books VALUES (1, 'Bytebase');", false)+ require.NoError(t, err)+ require.Equal(t, "readonly", got)+}++func TestResolveDataSourceIDUsesAdminForDocumentEngineAutomaticWriteQueryWhenAllowed(t *testing.T) {+ tests := []struct {+ name string+ engine storepb.Engine+ statement string+ }{+ {+ name: "MongoDB DML",+ engine: storepb.Engine_MONGODB,+ statement: `db.users.insertOne({name: "Bytebase"})`,+ },+ {+ name: "MongoDB DDL",+ engine: storepb.Engine_MONGODB,+ statement: `db.createCollection("users")`,+ },+ {+ name: "Elasticsearch DML",+ engine: storepb.Engine_ELASTICSEARCH,+ statement: "POST /users/_doc\n{\"name\":\"Bytebase\"}",+ },+ {+ name: "Elasticsearch DDL",+ engine: storepb.Engine_ELASTICSEARCH,+ statement: "PUT /users",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ instance := &store.InstanceMessage{+ Metadata: &storepb.Instance{+ Engine: tc.engine,+ DataSources: []*storepb.DataSource{+ {Id: "admin", Type: storepb.DataSourceType_ADMIN},+ {Id: "readonly", Type: storepb.DataSourceType_READ_ONLY},+ },+ },+ }++ got, err := resolveDataSourceID(context.Background(), instance, "", tc.statement, true)+ require.NoError(t, err)+ require.Equal(t, "admin", got)+ })+ }+}++func TestResolveDataSourceIDKeepsReadOnlyForDocumentEngineAutomaticReadQueryWhenAllowed(t *testing.T) {+ tests := []struct {+ name string+ engine storepb.Engine+ statement string+ }{+ {+ name: "MongoDB read",+ engine: storepb.Engine_MONGODB,+ statement: `db.users.find({})`,+ },+ {+ name: "Elasticsearch read",+ engine: storepb.Engine_ELASTICSEARCH,+ statement: "GET /users/_search",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ instance := &store.InstanceMessage{+ Metadata: &storepb.Instance{+ Engine: tc.engine,+ DataSources: []*storepb.DataSource{+ {Id: "admin", Type: storepb.DataSourceType_ADMIN},+ {Id: "readonly", Type: storepb.DataSourceType_READ_ONLY},+ },+ },+ }++ got, err := resolveDataSourceID(context.Background(), instance, "", tc.statement, true)+ require.NoError(t, err)+ require.Equal(t, "readonly", got)+ })+ }+}diff --git a/backend/tests/sql_query_data_source_test.go b/backend/tests/sql_query_data_source_test.goindex 97104efd12..58a7958f0c 100644--- a/backend/tests/sql_query_data_source_test.go+++ b/backend/tests/sql_query_data_source_test.go@@ -107,7 +107,31 @@ func TestSQLQueryDataSourceResolution(t *testing.T) {instanceID, err := common.GetInstanceID(instance.Name)a.NoError(err)stores := getStore(t, ctl.server)- instanceMessage, err := stores.GetInstance(ctx, &store.FindInstanceMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), ResourceID: &instanceID})+ workspaceID, err := stores.GetWorkspaceID(ctx)+ a.NoError(err)+ a.NotEmpty(workspaceID)+ _, err = ctl.orgPolicyServiceClient.CreatePolicy(ctx, connect.NewRequest(&v1pb.CreatePolicyRequest{+ Parent: common.FormatWorkspace(workspaceID),+ Policy: &v1pb.Policy{+ Type: v1pb.PolicyType_DATA_QUERY,+ Policy: &v1pb.Policy_QueryDataPolicy{+ QueryDataPolicy: &v1pb.QueryDataPolicy{+ AllowAdminDataSource: true,+ },+ },+ },+ }))+ a.NoError(err)++ queryResp, err = ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{+ Name: database.Name,+ Statement: "INSERT INTO books VALUES (2, 'Bytebase Admin');",+ }))+ a.NoError(err)+ a.Len(queryResp.Msg.Results, 1)+ a.Empty(queryResp.Msg.Results[0].Error)++ instanceMessage, err := stores.GetInstance(ctx, &store.FindInstanceMessage{Workspace: workspaceID, ResourceID: &instanceID})a.NoError(err)metadata := proto.CloneOf(instanceMessage.Metadata)var readOnly *storepb.DataSource@@ -124,7 +148,7 @@ func TestSQLQueryDataSourceResolution(t *testing.T) {metadata.DataSources = append(metadata.DataSources, readOnly)_, err = stores.UpdateInstance(ctx, &store.UpdateInstanceMessage{ResourceID: &instanceID,- Workspace: common.GetWorkspaceIDFromContext(ctx),+ Workspace: workspaceID,Metadata: metadata,})a.NoError(err)