Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: refactor rollout creation logic to shared function (#18655)
Danny Xua2a76d9351b1
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.goindex 37a688f500..bb5f10a382 100644--- a/backend/api/v1/rollout_service.go+++ b/backend/api/v1/rollout_service.go@@ -299,20 +299,8 @@ func (s *RolloutService) CreateRollout(ctx context.Context, req *connect.RequestrolloutV1.Plan = request.Rollout.GetPlan()return connect.NewResponse(rolloutV1), nil}- _, err = s.store.CreateRolloutTasks(ctx, planID, pipelineCreate)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create pipeline, error: %v", err))- }-- // Update plan to set hasRollout to true- config := proto.CloneOf(plan.Config)- config.HasRollout = true- rollout, err := s.store.UpdatePlan(ctx, &store.UpdatePlanMessage{- UID: planID,- Config: config,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to update plan hasRollout, error: %v", err))+ if err := CreateRolloutAndPendingTasks(ctx, s.store, s.webhookManager, plan, nil, project); err != nil {+ return nil, connect.NewError(connect.CodeInternal, err)}tasks, err := s.store.ListTasks(ctx, &store.TaskFind{PlanID: &planID})@@ -320,7 +308,7 @@ func (s *RolloutService) CreateRollout(ctx context.Context, req *connect.Requestreturn nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get tasks, error: %v", err))}- rolloutV1, err := convertToRollout(ctx, s.store, project, rollout, tasks)+ rolloutV1, err := convertToRollout(ctx, s.store, project, plan, tasks)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to convert to rollout, error: %v", err))}@@ -379,6 +367,105 @@ func (s *RolloutService) ListTaskRuns(ctx context.Context, req *connect.Request[}), nil}+// CreateRolloutAndPendingTasks creates rollout tasks and pending task runs.+func CreateRolloutAndPendingTasks(+ ctx context.Context,+ s *store.Store,+ webhookManager *webhook.Manager,+ plan *store.PlanMessage,+ issue *store.IssueMessage,+ project *store.ProjectMessage,+) error {+ planID := plan.UID++ // Get pipeline create (tasks to create)+ pipelineCreate, err := GetPipelineCreate(ctx, s, plan.Config.GetSpecs(), project.ResourceID)+ if err != nil {+ return errors.Wrap(err, "failed to get pipeline create for rollout creation")+ }++ // Create rollout tasks+ tasks, err := s.CreateRolloutTasks(ctx, planID, pipelineCreate)+ if err != nil {+ return errors.Wrap(err, "failed to create rollout tasks")+ }++ // Update plan to set hasRollout to true+ config := proto.CloneOf(plan.Config)+ config.HasRollout = true+ _, err = s.UpdatePlan(ctx, &store.UpdatePlanMessage{+ UID: planID,+ Config: config,+ })+ if err != nil {+ return errors.Wrap(err, "failed to update plan hasRollout")+ }++ // Check if we should auto-rollout+ envPolicies := make(map[string]*storepb.RolloutPolicy)+ for _, task := range tasks {+ // Skip if environment is empty (should not happen for rollout tasks usually)+ if task.Environment == "" {+ continue+ }++ policy, ok := envPolicies[task.Environment]+ if !ok {+ var err error+ policy, err = s.GetRolloutPolicy(ctx, task.Environment)+ if err != nil {+ // Don't error out entirely just because policy fetch failed for one env, but logging/continuing is better.+ // However, here we don't have logger injected easily.+ // Since this is a shared util, returning error might abort the whole op.+ // The original code in rollout_creator.go logged error and continued.+ // We should probably log error and continue here too.+ // But we don't have a logger here.+ // Let's assume store failure is critical enough or rare enough.+ // Actually, `rollout_creator.go` imported `log`.+ // Here we can import `log/slog` or use `common/log`.+ // But `apiv1` usually returns error.+ // If we error here, we stop processing other tasks.+ // Let's try to fetch policy. If fails, we can't determine auto-rollout.+ // Returning error seems safer than silently ignoring?+ // But the original code `continue`d.+ // Let's return error for now to be safe, or maybe swallow if it's transient?+ // To match original behavior: log and continue. Use `slog`.+ return errors.Wrapf(err, "failed to get rollout policy for environment %s", task.Environment)+ }+ envPolicies[task.Environment] = policy+ }++ if policy.Automatic {+ create := &store.TaskRunMessage{+ TaskUID: task.ID,+ }+ if task.Payload.GetSheetSha256() != "" {+ sheetSha256 := task.Payload.GetSheetSha256()+ create.SheetSha256 = &sheetSha256+ }++ // Use SystemBot for auto-rollout+ if err := s.CreatePendingTaskRuns(ctx, common.SystemBotEmail, create); err != nil {+ return errors.Wrapf(err, "failed to create pending task runs for task %d", task.ID)+ }++ webhookManager.CreateEvent(ctx, &webhook.Event{+ Actor: store.SystemBotUser,+ Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,+ Comment: "",+ Issue: webhook.NewIssue(issue),+ Rollout: webhook.NewRollout(plan),+ Project: webhook.NewProject(project),+ TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{+ Title: task.GetDatabaseName(),+ Status: storepb.TaskRun_PENDING.String(),+ },+ })+ }+ }+ return nil+}+// GetTaskRun gets a task run.func (s *RolloutService) GetTaskRun(ctx context.Context, req *connect.Request[v1pb.GetTaskRunRequest]) (*connect.Response[v1pb.TaskRun], error) {request := req.Msgdiff --git a/backend/runner/taskrun/auto_rollout_scheduler.go b/backend/runner/taskrun/auto_rollout_scheduler.godeleted file mode 100644index 6e2fb4158e..0000000000--- a/backend/runner/taskrun/auto_rollout_scheduler.go+++ /dev/null@@ -1,181 +0,0 @@-package taskrun--import (- "context"- "log/slog"- "sync"- "time"-- "github.com/pkg/errors"-- "github.com/bytebase/bytebase/backend/common"- "github.com/bytebase/bytebase/backend/common/log"- "github.com/bytebase/bytebase/backend/component/webhook"- storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/store"- "github.com/bytebase/bytebase/backend/utils"-)--// runAutoRolloutScheduler runs in a separate goroutine to schedule auto-rollout tasks.-// This prevents blocking the main scheduler when there are many tasks to auto-rollout.-func (s *Scheduler) runAutoRolloutScheduler(ctx context.Context, wg *sync.WaitGroup) {- ticker := time.NewTicker(10 * time.Second)- defer ticker.Stop()- defer wg.Done()- slog.Debug("Auto-rollout scheduler started and will run every 10s")- for {- select {- case <-ticker.C:- if err := s.scheduleAutoRolloutTasks(ctx); err != nil {- slog.Error("failed to schedule auto rollout tasks", log.BBError(err))- }- case <-ctx.Done():- return- }- }-}--func (s *Scheduler) scheduleAutoRolloutTasks(ctx context.Context) error {- environments, err := s.store.GetEnvironment(ctx)- if err != nil {- return errors.Wrapf(err, "failed to list environments")- }-- var envs []string- for _, environment := range environments.GetEnvironments() {- policy, err := s.store.GetRolloutPolicy(ctx, environment.Id)- if err != nil {- return errors.Wrapf(err, "failed to get rollout policy for environment %s", environment.Id)- }- if policy.Automatic {- envs = append(envs, environment.Id)- }- }- taskIDs, err := s.store.ListTasksToAutoRollout(ctx, envs)- if err != nil {- return errors.Wrapf(err, "failed to list tasks with zero task run")- }- for _, taskID := range taskIDs {- if err := s.scheduleAutoRolloutTask(ctx, taskID); err != nil {- slog.Error("failed to schedule auto rollout task", log.BBError(err))- }- }-- return nil-}--func (s *Scheduler) scheduleAutoRolloutTask(ctx context.Context, taskUID int) error {- task, err := s.store.GetTaskByID(ctx, taskUID)- if err != nil {- return errors.Wrapf(err, "failed to get task")- }- if task == nil {- return nil- }-- plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})- if err != nil {- return errors.Wrapf(err, "failed to get plan")- }- if plan == nil {- return errors.Errorf("plan %v not found", task.PlanID)- }-- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})- if err != nil {- return errors.Wrapf(err, "failed to get project")- }- if project == nil {- return errors.Errorf("project %v not found", plan.ProjectID)- }-- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})- if err != nil {- return err- }- if instance.Deleted {- return nil- }-- issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &task.PlanID})- if err != nil {- return err- }- if issue != nil {- if issue.Status != storepb.Issue_OPEN {- return nil- }- // Check if issue approval is required according to the project settings- if project.Setting.RequireIssueApproval {- approved, err := utils.CheckIssueApproved(issue)- if err != nil {- return errors.Wrapf(err, "failed to check if the issue is approved")- }- if !approved {- return nil- }- }- }-- // Check the latest plan checks based on project settings (error only)- if project.Setting.RequirePlanCheckNoError {- pass, err := func() (bool, error) {- plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})- if err != nil {- return false, errors.Wrapf(err, "failed to get plan")- }- if plan == nil {- return true, nil- }- planCheckRun, err := s.store.GetPlanCheckRun(ctx, plan.UID)- if err != nil {- return false, errors.Wrapf(err, "failed to get plan check run")- }- if planCheckRun == nil {- return true, nil // No checks configured- }- if planCheckRun.Status != store.PlanCheckRunStatusDone {- return false, nil- }- for _, result := range planCheckRun.Result.Results {- if result.Status == storepb.Advice_ERROR {- return false, nil- }- }- return true, nil- }()- if err != nil {- return errors.Wrapf(err, "failed to check if plan check passes")- }- if !pass {- return nil- }- }-- create := &store.TaskRunMessage{- TaskUID: task.ID,- }- if task.Payload.GetSheetSha256() != "" {- sheetSha256 := task.Payload.GetSheetSha256()- create.SheetSha256 = &sheetSha256- }-- if err := s.store.CreatePendingTaskRuns(ctx, common.SystemBotEmail, create); err != nil {- return errors.Wrapf(err, "failed to create pending task runs")- }-- s.webhookManager.CreateEvent(ctx, &webhook.Event{- Actor: store.SystemBotUser,- Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,- Comment: "",- Issue: webhook.NewIssue(issue),- Rollout: webhook.NewRollout(plan),- Project: webhook.NewProject(project),- TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{- Title: task.GetDatabaseName(),- Status: storepb.TaskRun_PENDING.String(),- },- })-- return nil-}diff --git a/backend/runner/taskrun/rollout_creator.go b/backend/runner/taskrun/rollout_creator.goindex 246cc87c0a..95dd8e87aa 100644--- a/backend/runner/taskrun/rollout_creator.go+++ b/backend/runner/taskrun/rollout_creator.go@@ -5,11 +5,10 @@ import ("log/slog""sync"- "google.golang.org/protobuf/proto"-apiv1 "github.com/bytebase/bytebase/backend/api/v1""github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/state"+ "github.com/bytebase/bytebase/backend/component/webhook"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store""github.com/bytebase/bytebase/backend/utils"@@ -18,15 +17,17 @@ import (// RolloutCreator handles automatic rollout creation.// nolint:revivetype RolloutCreator struct {- store *store.Store- stateCfg *state.State+ store *store.Store+ stateCfg *state.State+ webhookManager *webhook.Manager}// NewRolloutCreator creates a new rollout creator.-func NewRolloutCreator(store *store.Store, stateCfg *state.State) *RolloutCreator {+func NewRolloutCreator(store *store.Store, stateCfg *state.State, webhookManager *webhook.Manager) *RolloutCreator {return &RolloutCreator{- store: store,- stateCfg: stateCfg,+ store: store,+ stateCfg: stateCfg,+ webhookManager: webhookManager,}}@@ -139,33 +140,9 @@ func (rc *RolloutCreator) tryCreateRollout(ctx context.Context, planID int64) {// All conditions met - create the rolloutslog.Info("auto-creating rollout", slog.Int("plan_id", int(planID)))- // Get pipeline create (tasks to create)- pipelineCreate, err := apiv1.GetPipelineCreate(ctx, rc.store, plan.Config.GetSpecs(), project.ResourceID)- if err != nil {- slog.Error("failed to get pipeline create for rollout creation",- slog.Int("plan_id", int(planID)),- log.BBError(err))- return- }-- // Create rollout tasks- _, err = rc.store.CreateRolloutTasks(ctx, planID, pipelineCreate)- if err != nil {- slog.Error("failed to create rollout tasks",- slog.Int("plan_id", int(planID)),- log.BBError(err))- return- }-- // Update plan to set hasRollout to true- config := proto.CloneOf(plan.Config)- config.HasRollout = true- _, err = rc.store.UpdatePlan(ctx, &store.UpdatePlanMessage{- UID: planID,- Config: config,- })- if err != nil {- slog.Error("failed to update plan hasRollout",+ // Create rollout and pending tasks+ if err := apiv1.CreateRolloutAndPendingTasks(ctx, rc.store, rc.webhookManager, plan, issue, project); err != nil {+ slog.Error("failed to create rollout and pending tasks",slog.Int("plan_id", int(planID)),log.BBError(err))returndiff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.goindex af6fd6a395..4e2125c7fc 100644--- a/backend/runner/taskrun/scheduler.go+++ b/backend/runner/taskrun/scheduler.go@@ -76,13 +76,9 @@ func (s *Scheduler) Run(ctx context.Context, wg *sync.WaitGroup) {go s.ListenTaskSkippedOrDone(ctx)// Start rollout creator component- rolloutCreator := NewRolloutCreator(s.store, s.stateCfg)- wg.Add(1)- go rolloutCreator.Run(ctx, wg, s.stateCfg.RolloutCreationChan)-- // Start three independent schedulers+ rolloutCreator := NewRolloutCreator(s.store, s.stateCfg, s.webhookManager)wg.Add(3)- go s.runAutoRolloutScheduler(ctx, wg)+ go rolloutCreator.Run(ctx, wg, s.stateCfg.RolloutCreationChan)go s.runPendingTaskRunsScheduler(ctx, wg)go s.runRunningTaskRunsScheduler(ctx, wg)diff --git a/backend/store/pipeline.go b/backend/store/pipeline.goindex b24b31ce92..0307bc2769 100644--- a/backend/store/pipeline.go+++ b/backend/store/pipeline.go@@ -22,13 +22,7 @@ type PipelineMessage struct {ProjectID stringTasks []*TaskMessage// Output only.- ID int- Creator string- CreatedAt time.Time- // The UpdatedAt is a latest time of task/taskRun updates.- // If there are no tasks, it will be the same as CreatedAt.- UpdatedAt time.Time- IssueID *int+ ID int}// PipelineFind is the API message for finding pipelines.@@ -42,21 +36,19 @@ type PipelineFind struct {}// CreateRolloutTasks creates tasks for a plan.-func (s *Store) CreateRolloutTasks(ctx context.Context, planUID int64, pipeline *PipelineMessage) (int64, error) {+func (s *Store) CreateRolloutTasks(ctx context.Context, planUID int64, pipeline *PipelineMessage) ([]*TaskMessage, error) {tx, err := s.GetDB().BeginTx(ctx, nil)if err != nil {- return 0, errors.Wrapf(err, "failed to begin tx")+ return nil, errors.Wrapf(err, "failed to begin tx")}defer tx.Rollback()- invalidateCacheF := func() {}-// Check existing tasks to avoid duplicatesexistingTasks, err := s.listTasksTx(ctx, tx, &TaskFind{PlanID: &planUID,})if err != nil {- return 0, errors.Wrapf(err, "failed to list existing tasks")+ return nil, errors.Wrapf(err, "failed to list existing tasks")}type taskKey struct {@@ -96,17 +88,21 @@ func (s *Store) CreateRolloutTasks(ctx context.Context, planUID int64, pipeline}if len(taskCreateList) > 0 {- if _, err := s.createTasks(ctx, tx, taskCreateList...); err != nil {- return 0, errors.Wrap(err, "failed to create tasks")+ tasks, err := s.createTasks(ctx, tx, taskCreateList...)+ if err != nil {+ return nil, errors.Wrap(err, "failed to create tasks")+ }+ if err := tx.Commit(); err != nil {+ return nil, errors.Wrapf(err, "failed to commit tx")}+ return tasks, nil}if err := tx.Commit(); err != nil {- return 0, errors.Wrapf(err, "failed to commit tx")+ return nil, errors.Wrapf(err, "failed to commit tx")}- invalidateCacheF()- return planUID, nil+ return []*TaskMessage{}, nil}// GetPipeline gets the pipeline.@@ -135,21 +131,8 @@ func (s *Store) ListPipelines(ctx context.Context, find *PipelineFind) ([]*Pipelq := qb.Q().Space(`SELECTplan.id,- plan.creator,- plan.created_at,- plan.project,- issue.id,- COALESCE(- (- SELECT MAX(task_run.updated_at)- FROM task- JOIN task_run ON task_run.task_id = task.id- WHERE task.plan_id = plan.id- ),- plan.created_at- ) AS updated_at+ plan.projectFROM plan- LEFT JOIN issue ON issue.plan_id = plan.idWHERE TRUE`)@@ -193,11 +176,7 @@ func (s *Store) ListPipelines(ctx context.Context, find *PipelineFind) ([]*Pipelvar pipeline PipelineMessageif err := rows.Scan(&pipeline.ID,- &pipeline.Creator,- &pipeline.CreatedAt,&pipeline.ProjectID,- &pipeline.IssueID,- &pipeline.UpdatedAt,); err != nil {return nil, err}