Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: share plan database group expansion (#20740)
p0nyab93e0e2089a
diff --git a/backend/runner/approval/runner.go b/backend/runner/approval/runner.goindex 248592eed9..f0cbfe2361 100644--- a/backend/runner/approval/runner.go+++ b/backend/runner/approval/runner.go@@ -459,10 +459,19 @@ type specTarget struct {// unfoldDatabaseTargets unfolds database groups and returns all database targets.// If the targets list contains a single database group reference, it unfolds it to individual databases.// Otherwise, it returns the targets as-is.-func unfoldDatabaseTargets(ctx context.Context, stores *store.Store, dbTargets []string, projectID string, allDatabases []*store.DatabaseMessage) ([]string, error) {+func unfoldDatabaseTargets(ctx context.Context, stores *store.Store, dbTargets []string, projectID string, allDatabases []*store.DatabaseMessage, databaseGroup *v1pb.DatabaseGroup) ([]string, error) {// Check if this is a database group (single target)if len(dbTargets) == 1 {- if _, databaseGroupID, err := common.GetProjectIDDatabaseGroupID(dbTargets[0]); err == nil {+ target := dbTargets[0]+ if databaseGroup != nil && databaseGroup.Name == target {+ var unfolded []string+ for _, db := range databaseGroup.MatchedDatabases {+ unfolded = append(unfolded, db.Name)+ }+ return unfolded, nil+ }++ if _, databaseGroupID, err := common.GetProjectIDDatabaseGroupID(target); err == nil {// This is a database group - unfold itdatabaseGroup, err := stores.GetDatabaseGroup(ctx, &store.FindDatabaseGroupMessage{ResourceID: &databaseGroupID,@@ -492,11 +501,14 @@ func unfoldDatabaseTargets(ctx context.Context, stores *store.Store, dbTargets [}// unfoldSpecTargets unfolds database groups in specs and returns all database targets.-func unfoldSpecTargets(ctx context.Context, stores *store.Store, specs []*storepb.PlanConfig_Spec, projectID string) ([]specTarget, error) {+func unfoldSpecTargets(ctx context.Context, stores *store.Store, specs []*storepb.PlanConfig_Spec, projectID string, allDatabases []*store.DatabaseMessage, databaseGroup *v1pb.DatabaseGroup) ([]specTarget, error) {// Batch fetch all databases for the project- allDatabases, err := stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &projectID})- if err != nil {- return nil, errors.Wrapf(err, "failed to list databases for project %q", projectID)+ if allDatabases == nil {+ var err error+ allDatabases, err = stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &projectID})+ if err != nil {+ return nil, errors.Wrapf(err, "failed to list databases for project %q", projectID)+ }}// Build a map for quick lookup: instanceID/databaseName -> DatabaseMessage@@ -527,7 +539,7 @@ func unfoldSpecTargets(ctx context.Context, stores *store.Store, specs []*storep})case *storepb.PlanConfig_Spec_ChangeDatabaseConfig:- dbTargets, err := unfoldDatabaseTargets(ctx, stores, config.ChangeDatabaseConfig.Targets, projectID, allDatabases)+ dbTargets, err := unfoldDatabaseTargets(ctx, stores, config.ChangeDatabaseConfig.Targets, projectID, allDatabases, databaseGroup)if err != nil {return nil, err}@@ -544,7 +556,7 @@ func unfoldSpecTargets(ctx context.Context, stores *store.Store, specs []*storep}case *storepb.PlanConfig_Spec_ExportDataConfig:- dbTargets, err := unfoldDatabaseTargets(ctx, stores, config.ExportDataConfig.Targets, projectID, allDatabases)+ dbTargets, err := unfoldDatabaseTargets(ctx, stores, config.ExportDataConfig.Targets, projectID, allDatabases, databaseGroup)if err != nil {return nil, err}@@ -616,10 +628,28 @@ func buildCELVariablesForDatabaseChange(ctx context.Context, stores *store.Store}approvalInputVersion := plan.Config.GetApprovalInputVersion()- checksExpected, err := planChecksExpectedForApproval(ctx, stores, nil, plan)+ project, err := stores.GetProjectByResourceID(ctx, plan.ProjectID)+ if err != nil {+ return nil, approvalInputVersion, false, errors.Wrap(err, "failed to get project")+ }+ if project == nil {+ return nil, approvalInputVersion, false, errors.Errorf("project %s not found", plan.ProjectID)+ }++ allDatabases, err := stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &plan.ProjectID})+ if err != nil {+ return nil, approvalInputVersion, false, errors.Wrapf(err, "failed to list databases for project %q", plan.ProjectID)+ }++ databaseGroup, err := plancheck.GetDatabaseGroupForPlan(ctx, stores, plan, allDatabases)+ if err != nil {+ return nil, approvalInputVersion, false, err+ }+ checkTargets, err := plancheck.DeriveCheckTargets(ctx, stores, project, plan, databaseGroup)if err != nil {return nil, approvalInputVersion, false, errors.Wrap(err, "failed to derive plan check targets")}+ checksExpected := len(checkTargets) > 0planCheckRun, err := stores.GetPlanCheckRun(ctx, issue.ProjectID, plan.UID)if err != nil {@@ -695,7 +725,7 @@ func buildCELVariablesForDatabaseChange(ctx context.Context, stores *store.Store}// Unfold database groups and get all targets- targets, err := unfoldSpecTargets(ctx, stores, plan.Config.GetSpecs(), issue.ProjectID)+ targets, err := unfoldSpecTargets(ctx, stores, plan.Config.GetSpecs(), issue.ProjectID, allDatabases, databaseGroup)if err != nil {return nil, approvalInputVersion, false, errors.Wrap(err, "failed to unfold spec targets")}@@ -789,7 +819,7 @@ func planChecksExpectedForApproval(ctx context.Context, stores *store.Store, pro}}- databaseGroup, err := getDatabaseGroupForPlanApproval(ctx, stores, plan)+ databaseGroup, err := plancheck.GetDatabaseGroupForPlan(ctx, stores, plan, nil)if err != nil {return false, err}@@ -801,54 +831,6 @@ func planChecksExpectedForApproval(ctx context.Context, stores *store.Store, proreturn len(targets) > 0, nil}-func getDatabaseGroupForPlanApproval(ctx context.Context, stores *store.Store, plan *store.PlanMessage) (*v1pb.DatabaseGroup, error) {- for _, spec := range plan.Config.GetSpecs() {- cfg, ok := spec.Config.(*storepb.PlanConfig_Spec_ChangeDatabaseConfig)- if !ok {- continue- }- if len(cfg.ChangeDatabaseConfig.Targets) != 1 {- continue- }-- target := cfg.ChangeDatabaseConfig.Targets[0]- _, databaseGroupID, err := common.GetProjectIDDatabaseGroupID(target)- if err != nil {- continue- }-- dbGroup, err := stores.GetDatabaseGroup(ctx, &store.FindDatabaseGroupMessage{- ResourceID: &databaseGroupID,- ProjectIDs: []string{plan.ProjectID},- })- if err != nil {- return nil, errors.Wrapf(err, "failed to get database group %q", target)- }- if dbGroup == nil {- return nil, errors.Errorf("database group %q not found", target)- }-- allDatabases, err := stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &plan.ProjectID})- if err != nil {- return nil, errors.Wrapf(err, "failed to list databases for project %q", plan.ProjectID)- }-- matchedDatabases, err := utils.GetMatchedDatabasesInDatabaseGroup(ctx, dbGroup, allDatabases)- if err != nil {- return nil, errors.Wrapf(err, "failed to get matched databases for group %q", databaseGroupID)- }-- result := &v1pb.DatabaseGroup{Name: target}- for _, db := range matchedDatabases {- result.MatchedDatabases = append(result.MatchedDatabases, &v1pb.DatabaseGroup_Database{- Name: common.FormatDatabase(db.InstanceID, db.DatabaseName),- })- }- return result, nil- }- return nil, nil-}-// buildCELVariablesForDataExport builds CEL variables for DATABASE_EXPORT issues.func buildCELVariablesForDataExport(ctx context.Context, stores *store.Store, issue *store.IssueMessage) ([]map[string]any, bool, error) {if issue.PlanUID == nil {@@ -863,7 +845,7 @@ func buildCELVariablesForDataExport(ctx context.Context, stores *store.Store, is}// Unfold database groups and get all targets (only EXPORT_DATA targets)- targets, err := unfoldSpecTargets(ctx, stores, plan.Config.GetSpecs(), issue.ProjectID)+ targets, err := unfoldSpecTargets(ctx, stores, plan.Config.GetSpecs(), issue.ProjectID, nil, nil)if err != nil {return nil, false, errors.Wrap(err, "failed to unfold spec targets")}diff --git a/backend/runner/approval/runner_test.go b/backend/runner/approval/runner_test.goindex c008f3993b..64281821ba 100644--- a/backend/runner/approval/runner_test.go+++ b/backend/runner/approval/runner_test.go@@ -11,6 +11,7 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/testcontainer"storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/migrator""github.com/bytebase/bytebase/backend/store")@@ -382,6 +383,30 @@ func TestIsPlanCheckRunCurrentForApprovalInputVersion(t *testing.T) {}}+func TestUnfoldDatabaseTargetsUsesResolvedDatabaseGroup(t *testing.T) {+ database := &store.DatabaseMessage{+ InstanceID: "instances/prod",+ DatabaseName: "app",+ }+ databaseGroup := &v1pb.DatabaseGroup{+ Name: "projects/project/databaseGroups/group",+ MatchedDatabases: []*v1pb.DatabaseGroup_Database{{+ Name: common.FormatDatabase(database.InstanceID, database.DatabaseName),+ }},+ }++ got, err := unfoldDatabaseTargets(+ context.Background(),+ nil,+ []string{databaseGroup.Name},+ "project",+ []*store.DatabaseMessage{database},+ databaseGroup,+ )+ require.NoError(t, err)+ require.Equal(t, []string{common.FormatDatabase(database.InstanceID, database.DatabaseName)}, got)+}+func TestBuildCELVariablesForDatabaseChangeCreatesMissingPlanCheckRun(t *testing.T) {ctx := context.Background()s := setupApprovalRunnerStore(ctx, t)diff --git a/backend/runner/plancheck/database_group.go b/backend/runner/plancheck/database_group.gonew file mode 100644index 0000000000..d498da0750--- /dev/null+++ b/backend/runner/plancheck/database_group.go@@ -0,0 +1,64 @@+package plancheck++import (+ "context"++ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/store"+ "github.com/bytebase/bytebase/backend/utils"+)++// GetDatabaseGroupForPlan checks if the plan targets a database group and returns it with matched databases.+// Returns nil if the plan does not target a database group.+func GetDatabaseGroupForPlan(ctx context.Context, stores *store.Store, plan *store.PlanMessage, allDatabases []*store.DatabaseMessage) (*v1pb.DatabaseGroup, error) {+ for _, spec := range plan.Config.GetSpecs() {+ cfg := spec.GetChangeDatabaseConfig()+ if cfg == nil {+ continue+ }+ if len(cfg.Targets) != 1 {+ continue+ }++ target := cfg.Targets[0]+ _, databaseGroupID, err := common.GetProjectIDDatabaseGroupID(target)+ if err != nil {+ continue+ }++ dbGroup, err := stores.GetDatabaseGroup(ctx, &store.FindDatabaseGroupMessage{+ ResourceID: &databaseGroupID,+ ProjectIDs: []string{plan.ProjectID},+ })+ if err != nil {+ return nil, errors.Wrapf(err, "failed to get database group %q", target)+ }+ if dbGroup == nil {+ return nil, errors.Errorf("database group %q not found", target)+ }++ if allDatabases == nil {+ allDatabases, err = stores.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &plan.ProjectID})+ if err != nil {+ return nil, errors.Wrapf(err, "failed to list databases for project %q", plan.ProjectID)+ }+ }++ matchedDatabases, err := utils.GetMatchedDatabasesInDatabaseGroup(ctx, dbGroup, allDatabases)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to get matched databases for group %q", databaseGroupID)+ }++ result := &v1pb.DatabaseGroup{Name: target}+ for _, db := range matchedDatabases {+ result.MatchedDatabases = append(result.MatchedDatabases, &v1pb.DatabaseGroup_Database{+ Name: common.FormatDatabase(db.InstanceID, db.DatabaseName),+ })+ }+ return result, nil+ }+ return nil, nil+}diff --git a/backend/runner/plancheck/scheduler.go b/backend/runner/plancheck/scheduler.goindex 75cc9378a8..b774bbfa47 100644--- a/backend/runner/plancheck/scheduler.go+++ b/backend/runner/plancheck/scheduler.go@@ -10,14 +10,11 @@ import ("github.com/pkg/errors"- "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/bus""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/store"- "github.com/bytebase/bytebase/backend/utils")const (@@ -130,7 +127,7 @@ func (s *Scheduler) runPlanCheckRun(ctx context.Context, projectID string, uid i}// Get database group if needed (for spec expansion)- databaseGroup, err := s.getDatabaseGroupForPlan(ctxWithCancel, plan)+ databaseGroup, err := GetDatabaseGroupForPlan(ctxWithCancel, s.store, plan, nil)if err != nil {s.markPlanCheckRunFailed(ctxWithCancel, projectID, uid, approvalInputVersion, err.Error())return@@ -250,60 +247,3 @@ func (s *Scheduler) markPlanCheckRunCanceled(ctx context.Context, projectID strislog.Int64("claimed_approval_input_version", approvalInputVersion))}}--// getDatabaseGroupForPlan checks if the plan targets a database group and returns it with matched databases.-// Returns nil if the plan does not target a database group.-func (s *Scheduler) getDatabaseGroupForPlan(ctx context.Context, plan *store.PlanMessage) (*v1pb.DatabaseGroup, error) {- for _, spec := range plan.Config.Specs {- cfg, ok := spec.Config.(*storepb.PlanConfig_Spec_ChangeDatabaseConfig)- if !ok {- continue- }- if len(cfg.ChangeDatabaseConfig.Targets) != 1 {- continue- }-- target := cfg.ChangeDatabaseConfig.Targets[0]- _, databaseGroupID, err := common.GetProjectIDDatabaseGroupID(target)- if err != nil {- // Not a database group reference, skip- continue- }-- // Found a database group reference - fetch and expand it- dbGroup, err := s.store.GetDatabaseGroup(ctx, &store.FindDatabaseGroupMessage{- ResourceID: &databaseGroupID,- ProjectIDs: []string{plan.ProjectID},- })- if err != nil {- return nil, errors.Wrapf(err, "failed to get database group %q", target)- }- if dbGroup == nil {- return nil, errors.Errorf("database group %q not found", target)- }-- // Get all databases in the project to compute matches- allDatabases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &plan.ProjectID})- if err != nil {- return nil, errors.Wrapf(err, "failed to list databases for project %q", plan.ProjectID)- }-- // Compute matched databases using CEL expression- matchedDatabases, err := utils.GetMatchedDatabasesInDatabaseGroup(ctx, dbGroup, allDatabases)- if err != nil {- return nil, errors.Wrapf(err, "failed to get matched databases for group %q", databaseGroupID)- }-- // Convert to v1pb.DatabaseGroup format- result := &v1pb.DatabaseGroup{- Name: target,- }- for _, db := range matchedDatabases {- result.MatchedDatabases = append(result.MatchedDatabases, &v1pb.DatabaseGroup_Database{- Name: common.FormatDatabase(db.InstanceID, db.DatabaseName),- })- }- return result, nil- }- return nil, nil-}