Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: avoid eager database listing for approval checks (#20741)
p0nyc7d2c639d8c7
diff --git a/backend/runner/approval/runner.go b/backend/runner/approval/runner.goindex f0cbfe2361..fa5e2543b2 100644--- a/backend/runner/approval/runner.go+++ b/backend/runner/approval/runner.go@@ -501,6 +501,8 @@ func unfoldDatabaseTargets(ctx context.Context, stores *store.Store, dbTargets [}// unfoldSpecTargets unfolds database groups in specs and returns all database targets.+// allDatabases must be nil or the full unfiltered project database list; callers pass it+// through to keep database-group matching and direct database lookup on the same snapshot.func unfoldSpecTargets(ctx context.Context, stores *store.Store, specs []*storepb.PlanConfig_Spec, projectID string, allDatabases []*store.DatabaseMessage, databaseGroup *v1pb.DatabaseGroup) ([]specTarget, error) {// Batch fetch all databases for the projectif allDatabases == nil {@@ -636,14 +638,17 @@ func buildCELVariablesForDatabaseChange(ctx context.Context, stores *store.Storereturn nil, approvalInputVersion, false, errors.Errorf("project %s not found", plan.ProjectID)}- allDatabases, err := stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &plan.ProjectID})- if err != nil {- return nil, approvalInputVersion, false, errors.Wrapf(err, "failed to list databases for project %q", plan.ProjectID)- }-- databaseGroup, err := plancheck.GetDatabaseGroupForPlan(ctx, stores, plan, allDatabases)- if err != nil {- return nil, approvalInputVersion, false, err+ var allDatabases []*store.DatabaseMessage+ var databaseGroup *v1pb.DatabaseGroup+ if plancheck.HasDatabaseGroupTarget(plan.Config.GetSpecs()) {+ allDatabases, err = stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &plan.ProjectID})+ if err != nil {+ return nil, approvalInputVersion, false, errors.Wrapf(err, "failed to list databases for project %q", plan.ProjectID)+ }+ databaseGroup, err = plancheck.GetDatabaseGroupForPlan(ctx, stores, plan, allDatabases)+ if err != nil {+ return nil, approvalInputVersion, false, err+ }}checkTargets, err := plancheck.DeriveCheckTargets(ctx, stores, project, plan, databaseGroup)if err != nil {@@ -807,30 +812,6 @@ func buildCELVariablesForDatabaseChange(ctx context.Context, stores *store.Storereturn celVarsList, approvalInputVersion, true, nil}-func planChecksExpectedForApproval(ctx context.Context, stores *store.Store, project *store.ProjectMessage, plan *store.PlanMessage) (bool, error) {- if project == nil {- var err error- project, err = stores.GetProjectByResourceID(ctx, plan.ProjectID)- if err != nil {- return false, errors.Wrap(err, "failed to get project")- }- if project == nil {- return false, errors.Errorf("project %s not found", plan.ProjectID)- }- }-- databaseGroup, err := plancheck.GetDatabaseGroupForPlan(ctx, stores, plan, nil)- if err != nil {- return false, err- }-- targets, err := plancheck.DeriveCheckTargets(ctx, stores, project, plan, databaseGroup)- if err != nil {- return false, err- }- return len(targets) > 0, nil-}-// buildCELVariablesForDataExport builds CEL variables for DATABASE_EXPORT issues.func buildCELVariablesForDataExport(ctx context.Context, stores *store.Store, issue *store.IssueMessage) ([]map[string]any, bool, error) {if issue.PlanUID == nil {diff --git a/backend/runner/approval/runner_test.go b/backend/runner/approval/runner_test.goindex 64281821ba..71ae5dc134 100644--- a/backend/runner/approval/runner_test.go+++ b/backend/runner/approval/runner_test.go@@ -13,6 +13,7 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/migrator"+ "github.com/bytebase/bytebase/backend/runner/plancheck""github.com/bytebase/bytebase/backend/store")@@ -240,78 +241,39 @@ func TestBuildStatementSummaryResultMapUsesSheetSHA256(t *testing.T) {}].GetSqlSummaryReport().GetAffectedRows())}-func TestPlanChecksExpectedForApproval(t *testing.T) {- tests := []struct {- name string- plan *store.PlanMessage- want bool- }{- {- name: "create database does not expect checks",- plan: &store.PlanMessage{- ProjectID: "project",- Config: &storepb.PlanConfig{- Specs: []*storepb.PlanConfig_Spec{{- Config: &storepb.PlanConfig_Spec_CreateDatabaseConfig{- CreateDatabaseConfig: &storepb.PlanConfig_CreateDatabaseConfig{},- },- }},+func TestDeriveCheckTargetsSkipsCreateDatabaseAndReleaseSpecs(t *testing.T) {+ project := &store.ProjectMessage{ResourceID: "project"}+ plan := &store.PlanMessage{+ ProjectID: "project",+ Config: &storepb.PlanConfig{+ Specs: []*storepb.PlanConfig_Spec{+ {+ Config: &storepb.PlanConfig_Spec_CreateDatabaseConfig{+ CreateDatabaseConfig: &storepb.PlanConfig_CreateDatabaseConfig{},+ },},- },- },- {- name: "release backed change does not expect checks",- plan: &store.PlanMessage{- ProjectID: "project",- Config: &storepb.PlanConfig{- Specs: []*storepb.PlanConfig_Spec{{- Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{- ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{- Release: "projects/project/releases/release",- },+ {+ Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{+ Release: "projects/project/releases/release",},- }},- },- },- },- {- name: "empty expanded targets do not expect checks",- plan: &store.PlanMessage{- ProjectID: "project",- Config: &storepb.PlanConfig{- Specs: []*storepb.PlanConfig_Spec{{- Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{- ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{},- },- }},+ },},- },- },- {- name: "change target expects checks",- plan: &store.PlanMessage{- ProjectID: "project",- Config: &storepb.PlanConfig{- Specs: []*storepb.PlanConfig_Spec{{- Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{- ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{- Targets: []string{"instances/prod/databases/app"},- },+ {+ Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{+ Targets: []string{"instances/prod/databases/app"},},- }},+ },},},- want: true,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := planChecksExpectedForApproval(context.Background(), nil, &store.ProjectMessage{ResourceID: "project"}, tt.plan)- require.NoError(t, err)- require.Equal(t, tt.want, got)- })- }+ got, err := plancheck.DeriveCheckTargets(context.Background(), nil, project, plan, nil)+ require.NoError(t, err)+ require.Len(t, got, 1)+ require.Equal(t, "instances/prod/databases/app", got[0].Target)}func TestIsPlanCheckRunCurrentForApprovalInputVersion(t *testing.T) {@@ -385,7 +347,7 @@ func TestIsPlanCheckRunCurrentForApprovalInputVersion(t *testing.T) {func TestUnfoldDatabaseTargetsUsesResolvedDatabaseGroup(t *testing.T) {database := &store.DatabaseMessage{- InstanceID: "instances/prod",+ InstanceID: "prod",DatabaseName: "app",}databaseGroup := &v1pb.DatabaseGroup{@@ -397,6 +359,7 @@ func TestUnfoldDatabaseTargetsUsesResolvedDatabaseGroup(t *testing.T) {got, err := unfoldDatabaseTargets(context.Background(),+ // nil store is intentional: the resolved-group path must not touch the store.nil,[]string{databaseGroup.Name},"project",@@ -407,6 +370,23 @@ func TestUnfoldDatabaseTargetsUsesResolvedDatabaseGroup(t *testing.T) {require.Equal(t, []string{common.FormatDatabase(database.InstanceID, database.DatabaseName)}, got)}+func TestUnfoldDatabaseTargetsFallsBackWhenResolvedGroupNameDiffers(t *testing.T) {+ ctx := context.Background()+ s := setupApprovalRunnerStore(ctx, t)+ allDatabases := setupApprovalDatabaseGroupFixture(ctx, t, s)++ got, err := unfoldDatabaseTargets(+ ctx,+ s,+ []string{"projects/project-a/databaseGroups/group"},+ "project-a",+ allDatabases,+ &v1pb.DatabaseGroup{Name: "projects/project-a/databaseGroups/other"},+ )+ require.NoError(t, err)+ require.Equal(t, []string{"instances/prod/databases/app"}, got)+}+func TestBuildCELVariablesForDatabaseChangeCreatesMissingPlanCheckRun(t *testing.T) {ctx := context.Background()s := setupApprovalRunnerStore(ctx, t)@@ -477,3 +457,36 @@ func setupApprovalRunnerStore(ctx context.Context, t *testing.T) *store.Store {t.Cleanup(func() { require.NoError(t, s.Close()) })return s}++func setupApprovalDatabaseGroupFixture(ctx context.Context, t *testing.T, s *store.Store) []*store.DatabaseMessage {+ t.Helper()++ _, err := s.CreateInstance(ctx, &store.InstanceMessage{+ ResourceID: "prod",+ Workspace: "default",+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_POSTGRES,+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},+ },+ })+ require.NoError(t, err)+ _, err = s.UpsertDatabase(ctx, &store.DatabaseMessage{+ ProjectID: "project-a",+ InstanceID: "prod",+ DatabaseName: "app",+ Metadata: &storepb.DatabaseMetadata{Labels: map[string]string{}},+ })+ require.NoError(t, err)+ _, err = s.CreateDatabaseGroup(ctx, &store.DatabaseGroupMessage{+ ProjectID: "project-a",+ ResourceID: "group",+ Title: "group",+ Expression: &expr.Expr{Expression: `resource.database_name == "app"`},+ })+ require.NoError(t, err)++ projectID := "project-a"+ allDatabases, err := s.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &projectID})+ require.NoError(t, err)+ return allDatabases+}diff --git a/backend/runner/plancheck/database_group.go b/backend/runner/plancheck/database_group.goindex d498da0750..800d4796b1 100644--- a/backend/runner/plancheck/database_group.go+++ b/backend/runner/plancheck/database_group.go@@ -6,6 +6,7 @@ import ("github.com/pkg/errors""github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/store""github.com/bytebase/bytebase/backend/utils"@@ -13,52 +14,66 @@ import (// GetDatabaseGroupForPlan checks if the plan targets a database group and returns it with matched databases.// Returns nil if the plan does not target a database group.+// allDatabases must be nil or the full unfiltered database list for the plan's project;+// a filtered list would silently make the group expansion incomplete.func GetDatabaseGroupForPlan(ctx context.Context, stores *store.Store, plan *store.PlanMessage, allDatabases []*store.DatabaseMessage) (*v1pb.DatabaseGroup, error) {- for _, spec := range plan.Config.GetSpecs() {- cfg := spec.GetChangeDatabaseConfig()- if cfg == nil {- continue- }- if len(cfg.Targets) != 1 {- continue- }+ target, databaseGroupID, ok := findDatabaseGroupTarget(plan.Config.GetSpecs())+ if !ok {+ return nil, nil+ }- target := cfg.Targets[0]- _, databaseGroupID, err := common.GetProjectIDDatabaseGroupID(target)+ dbGroup, err := stores.GetDatabaseGroup(ctx, &store.FindDatabaseGroupMessage{+ ResourceID: &databaseGroupID,+ ProjectIDs: []string{plan.ProjectID},+ })+ if err != nil {+ return nil, errors.Wrapf(err, "failed to get database group %q", target)+ }+ if dbGroup == nil {+ return nil, errors.Errorf("database group %q not found", target)+ }++ if allDatabases == nil {+ allDatabases, err = stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &plan.ProjectID})if err != nil {- continue+ return nil, errors.Wrapf(err, "failed to list databases for project %q", plan.ProjectID)}+ }++ matchedDatabases, err := utils.GetMatchedDatabasesInDatabaseGroup(ctx, dbGroup, allDatabases)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to get matched databases for group %q", databaseGroupID)+ }- dbGroup, err := stores.GetDatabaseGroup(ctx, &store.FindDatabaseGroupMessage{- ResourceID: &databaseGroupID,- ProjectIDs: []string{plan.ProjectID},+ result := &v1pb.DatabaseGroup{Name: target}+ for _, db := range matchedDatabases {+ result.MatchedDatabases = append(result.MatchedDatabases, &v1pb.DatabaseGroup_Database{+ Name: common.FormatDatabase(db.InstanceID, db.DatabaseName),})- if err != nil {- return nil, errors.Wrapf(err, "failed to get database group %q", target)- }- if dbGroup == nil {- return nil, errors.Errorf("database group %q not found", target)- }+ }+ return result, nil+}- if allDatabases == nil {- allDatabases, err = stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &plan.ProjectID})- if err != nil {- return nil, errors.Wrapf(err, "failed to list databases for project %q", plan.ProjectID)- }- }+// HasDatabaseGroupTarget reports whether any change-database spec targets a database group.+// It only parses target resource names and does not touch the store.+func HasDatabaseGroupTarget(specs []*storepb.PlanConfig_Spec) bool {+ _, _, ok := findDatabaseGroupTarget(specs)+ return ok+}- matchedDatabases, err := utils.GetMatchedDatabasesInDatabaseGroup(ctx, dbGroup, allDatabases)- if err != nil {- return nil, errors.Wrapf(err, "failed to get matched databases for group %q", databaseGroupID)+func findDatabaseGroupTarget(specs []*storepb.PlanConfig_Spec) (string, string, bool) {+ for _, spec := range specs {+ cfg := spec.GetChangeDatabaseConfig()+ if cfg == nil || len(cfg.Targets) != 1 {+ continue}- result := &v1pb.DatabaseGroup{Name: target}- for _, db := range matchedDatabases {- result.MatchedDatabases = append(result.MatchedDatabases, &v1pb.DatabaseGroup_Database{- Name: common.FormatDatabase(db.InstanceID, db.DatabaseName),- })+ target := cfg.Targets[0]+ _, databaseGroupID, err := common.GetProjectIDDatabaseGroupID(target)+ if err != nil {+ continue}- return result, nil+ return target, databaseGroupID, true}- return nil, nil+ return "", "", false}diff --git a/backend/runner/plancheck/database_group_test.go b/backend/runner/plancheck/database_group_test.gonew file mode 100644index 0000000000..c52560f0da--- /dev/null+++ b/backend/runner/plancheck/database_group_test.go@@ -0,0 +1,160 @@+package plancheck++import (+ "context"+ "fmt"+ "testing"++ "github.com/stretchr/testify/require"+ "google.golang.org/genproto/googleapis/type/expr"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/migrator"+ "github.com/bytebase/bytebase/backend/store"+)++func TestHasDatabaseGroupTarget(t *testing.T) {+ tests := []struct {+ name string+ specs []*storepb.PlanConfig_Spec+ want bool+ }{+ {+ name: "direct database target",+ specs: []*storepb.PlanConfig_Spec{{+ Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{+ Targets: []string{"instances/prod/databases/app"},+ },+ },+ }},+ },+ {+ name: "multiple targets are not a database group target",+ specs: []*storepb.PlanConfig_Spec{{+ Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{+ Targets: []string{"projects/project-a/databaseGroups/group", "instances/prod/databases/app"},+ },+ },+ }},+ },+ {+ name: "database group target",+ specs: []*storepb.PlanConfig_Spec{{+ Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{+ Targets: []string{"projects/project-a/databaseGroups/group"},+ },+ },+ }},+ want: true,+ },+ {+ name: "export data group target does not require plan check group expansion",+ specs: []*storepb.PlanConfig_Spec{{+ Config: &storepb.PlanConfig_Spec_ExportDataConfig{+ ExportDataConfig: &storepb.PlanConfig_ExportDataConfig{+ Targets: []string{"projects/project-a/databaseGroups/group"},+ },+ },+ }},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ require.Equal(t, tt.want, HasDatabaseGroupTarget(tt.specs))+ })+ }+}++func TestGetDatabaseGroupForPlanMatchesDatabases(t *testing.T) {+ ctx := context.Background()+ s := setupPlancheckStore(ctx, t)+ setupPlancheckDatabaseGroupFixture(ctx, t, s)++ target := "projects/project-a/databaseGroups/group"+ got, err := GetDatabaseGroupForPlan(ctx, s, &store.PlanMessage{+ ProjectID: "project-a",+ Config: &storepb.PlanConfig{+ Specs: []*storepb.PlanConfig_Spec{{+ Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{+ Targets: []string{target},+ },+ },+ }},+ },+ }, nil)+ require.NoError(t, err)+ require.NotNil(t, got)+ require.Equal(t, target, got.Name)+ require.Equal(t, []string{common.FormatDatabase("prod", "app")}, databaseNames(got.MatchedDatabases))+}++func setupPlancheckStore(ctx context.Context, t *testing.T) *store.Store {+ t.Helper()++ container := testcontainer.GetTestPgContainer(ctx, t)+ t.Cleanup(func() { container.Close(ctx) })++ db := container.GetDB()+ require.NoError(t, migrator.MigrateSchema(ctx, db))++ _, err := db.ExecContext(ctx, `+ INSERT INTO workspace (resource_id) VALUES ('default');+ INSERT INTO project (resource_id, workspace, name) VALUES ('project-a', 'default', 'Project A');+ `)+ require.NoError(t, err)++ pgURL := fmt.Sprintf(+ "host=%s port=%s user=postgres password=root-password database=postgres",+ container.GetHost(), container.GetPort(),+ )+ s, err := store.New(ctx, pgURL, false)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, s.Close()) })+ return s+}++func setupPlancheckDatabaseGroupFixture(ctx context.Context, t *testing.T, s *store.Store) {+ t.Helper()++ _, err := s.CreateInstance(ctx, &store.InstanceMessage{+ ResourceID: "prod",+ Workspace: "default",+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_POSTGRES,+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},+ },+ })+ require.NoError(t, err)+ for _, databaseName := range []string{"app", "audit"} {+ _, err = s.UpsertDatabase(ctx, &store.DatabaseMessage{+ ProjectID: "project-a",+ InstanceID: "prod",+ DatabaseName: databaseName,+ Metadata: &storepb.DatabaseMetadata{Labels: map[string]string{}},+ })+ require.NoError(t, err)+ }+ _, err = s.CreateDatabaseGroup(ctx, &store.DatabaseGroupMessage{+ ProjectID: "project-a",+ ResourceID: "group",+ Title: "group",+ Expression: &expr.Expr{Expression: `resource.database_name == "app"`},+ })+ require.NoError(t, err)+}++func databaseNames(databases []*v1pb.DatabaseGroup_Database) []string {+ var names []string+ for _, database := range databases {+ names = append(names, database.Name)+ }+ return names+}