Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: split task scheduler into independent schedulers (#18517)

Danny Xu47e8f8e52544
diff --git a/backend/runner/taskrun/auto_rollout_scheduler.go b/backend/runner/taskrun/auto_rollout_scheduler.go
new file mode 100644
index 0000000000..d85ec24d99
--- /dev/null
+++ b/backend/runner/taskrun/auto_rollout_scheduler.go
@@ -0,0 +1,182 @@
+package taskrun
+
+import (
+ "context"
+ "log/slog"
+ "sync"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/component/webhook"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/store"
+ "github.com/bytebase/bytebase/backend/utils"
+)
+
+// runAutoRolloutScheduler runs in a separate goroutine to schedule auto-rollout tasks.
+// This prevents blocking the main scheduler when there are many tasks to auto-rollout.
+func (s *Scheduler) runAutoRolloutScheduler(ctx context.Context, wg *sync.WaitGroup) {
+ ticker := time.NewTicker(10 * time.Second)
+ defer ticker.Stop()
+ defer wg.Done()
+ slog.Debug("Auto-rollout scheduler started and will run every 10s")
+ for {
+ select {
+ case <-ticker.C:
+ if err := s.scheduleAutoRolloutTasks(ctx); err != nil {
+ slog.Error("failed to schedule auto rollout tasks", log.BBError(err))
+ }
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func (s *Scheduler) scheduleAutoRolloutTasks(ctx context.Context) error {
+ environments, err := s.store.GetEnvironment(ctx)
+ if err != nil {
+ return errors.Wrapf(err, "failed to list environments")
+ }
+
+ var envs []string
+ for _, environment := range environments.GetEnvironments() {
+ policy, err := s.store.GetRolloutPolicy(ctx, environment.Id)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get rollout policy for environment %s", environment.Id)
+ }
+ if policy.Automatic {
+ envs = append(envs, environment.Id)
+ }
+ }
+ taskIDs, err := s.store.ListTasksToAutoRollout(ctx, envs)
+ if err != nil {
+ return errors.Wrapf(err, "failed to list tasks with zero task run")
+ }
+ for _, taskID := range taskIDs {
+ if err := s.scheduleAutoRolloutTask(ctx, taskID); err != nil {
+ slog.Error("failed to schedule auto rollout task", log.BBError(err))
+ }
+ }
+
+ return nil
+}
+
+func (s *Scheduler) scheduleAutoRolloutTask(ctx context.Context, taskUID int) error {
+ task, err := s.store.GetTaskByID(ctx, taskUID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get task")
+ }
+ if task == nil {
+ return nil
+ }
+
+ pipeline, err := s.store.GetPipelineByID(ctx, task.PipelineID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get pipeline")
+ }
+ if pipeline == nil {
+ return errors.Errorf("pipeline %v not found", task.PipelineID)
+ }
+
+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &pipeline.ProjectID})
+ if err != nil {
+ return errors.Wrapf(err, "failed to get project")
+ }
+ if project == nil {
+ return errors.Errorf("project %v not found", pipeline.ProjectID)
+ }
+
+ instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})
+ if err != nil {
+ return err
+ }
+ if instance.Deleted {
+ return nil
+ }
+
+ issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PipelineID: &task.PipelineID})
+ if err != nil {
+ return err
+ }
+ if issue != nil {
+ if issue.Status != storepb.Issue_OPEN {
+ return nil
+ }
+ // Check if issue approval is required according to the project settings
+ if project.Setting.RequireIssueApproval {
+ approved, err := utils.CheckIssueApproved(issue)
+ if err != nil {
+ return errors.Wrapf(err, "failed to check if the issue is approved")
+ }
+ if !approved {
+ return nil
+ }
+ }
+ }
+
+ // Check the latest plan checks based on project settings (error only)
+ if project.Setting.RequirePlanCheckNoError {
+ pass, err := func() (bool, error) {
+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{PipelineID: &task.PipelineID})
+ if err != nil {
+ return false, errors.Wrapf(err, "failed to get plan")
+ }
+ if plan == nil {
+ return true, nil
+ }
+ latestRuns, err := s.store.ListPlanCheckRuns(ctx, &store.FindPlanCheckRunMessage{
+ PlanUID: &plan.UID,
+ })
+ if err != nil {
+ return false, errors.Wrapf(err, "failed to list latest plan check runs")
+ }
+ for _, run := range latestRuns {
+ if run.Status != store.PlanCheckRunStatusDone {
+ return false, nil
+ }
+ for _, result := range run.Result.Results {
+ if result.Status == storepb.Advice_ERROR {
+ return false, nil
+ }
+ }
+ }
+ return true, nil
+ }()
+ if err != nil {
+ return errors.Wrapf(err, "failed to check if plan check passes")
+ }
+ if !pass {
+ return nil
+ }
+ }
+
+ create := &store.TaskRunMessage{
+ TaskUID: task.ID,
+ }
+ if task.Payload.GetSheetId() != 0 {
+ sheetUID := int(task.Payload.GetSheetId())
+ create.SheetUID = &sheetUID
+ }
+
+ if err := s.store.CreatePendingTaskRuns(ctx, common.SystemBotEmail, create); err != nil {
+ return errors.Wrapf(err, "failed to create pending task runs")
+ }
+
+ s.webhookManager.CreateEvent(ctx, &webhook.Event{
+ Actor: store.SystemBotUser,
+ Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,
+ Comment: "",
+ Issue: webhook.NewIssue(issue),
+ Rollout: webhook.NewRollout(pipeline),
+ Project: webhook.NewProject(project),
+ TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{
+ Title: task.GetDatabaseName(),
+ Status: storepb.TaskRun_PENDING.String(),
+ },
+ })
+
+ return nil
+}
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
new file mode 100644
index 0000000000..a1253308e3
--- /dev/null
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -0,0 +1,129 @@
+package taskrun
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "sync"
+ "time"
+
+ "github.com/pkg/errors"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+// runPendingTaskRunsScheduler runs in a separate goroutine to schedule pending task runs.
+func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.WaitGroup) {
+ ticker := time.NewTicker(taskSchedulerInterval)
+ defer ticker.Stop()
+ defer wg.Done()
+ slog.Debug(fmt.Sprintf("Pending task runs scheduler started and will run every %v", taskSchedulerInterval))
+ for {
+ select {
+ case <-ticker.C:
+ if err := s.schedulePendingTaskRuns(ctx); err != nil {
+ slog.Error("failed to schedule pending task runs", log.BBError(err))
+ }
+ case <-s.stateCfg.TaskRunTickleChan:
+ if err := s.schedulePendingTaskRuns(ctx); err != nil {
+ slog.Error("failed to schedule pending task runs", log.BBError(err))
+ }
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
+ taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
+ Status: &[]storepb.TaskRun_Status{storepb.TaskRun_PENDING},
+ })
+ if err != nil {
+ return errors.Wrapf(err, "failed to list pending tasks")
+ }
+ for _, taskRun := range taskRuns {
+ if err := s.schedulePendingTaskRun(ctx, taskRun); err != nil {
+ slog.Error("failed to schedule pending task run", log.BBError(err))
+ }
+ }
+
+ return nil
+}
+
+func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {
+ // here, we move pending taskruns to running taskruns which means they are ready to be executed.
+ // pending taskruns remain pending if
+ // 1. taskRun.RunAt not met.
+ // 2. for versioned tasks, there are other versioned tasks on the same database with
+ // a smaller version not finished yet. we need to wait for those first.
+ task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get task")
+ }
+ if taskRun.RunAt != nil && time.Now().Before(*taskRun.RunAt) {
+ return nil
+ }
+
+ doSchedule, err := func() (bool, error) {
+ if task.DatabaseName == nil {
+ return true, nil
+ }
+
+ schemaVersion := task.Payload.GetSchemaVersion()
+ if schemaVersion == "" {
+ return true, nil
+ }
+
+ maybeTaskID, err := s.store.FindBlockingTaskByVersion(ctx, task.PipelineID, task.InstanceID, *task.DatabaseName, schemaVersion)
+ if err != nil {
+ return false, errors.Wrapf(err, "failed to find blocking versioned tasks")
+ }
+ if maybeTaskID != nil {
+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
+ ReportTime: timestamppb.Now(),
+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{
+ Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
+ TaskUid: int32(*maybeTaskID),
+ },
+ },
+ })
+ return false, nil
+ }
+ s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)
+ return true, nil
+ }()
+ if err != nil {
+ return errors.Wrapf(err, "failed to check blocking versioned tasks")
+ }
+ if !doSchedule {
+ return nil
+ }
+
+ if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{
+ ID: taskRun.ID,
+ Updater: common.SystemBotEmail,
+ Status: storepb.TaskRun_RUNNING,
+ }); err != nil {
+ return errors.Wrapf(err, "failed to update task run status to running")
+ }
+ s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{
+ Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,
+ TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{
+ Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_WAITING,
+ },
+ })
+
+ // Tickle the running scheduler to immediately pick up this newly RUNNING task
+ // This prevents delays when user clicks "Run stage" in the UI
+ select {
+ case s.stateCfg.TaskRunTickleChan <- 0:
+ default:
+ // Channel is full, running scheduler will pick it up on next tick
+ }
+
+ return nil
+}
diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.go
new file mode 100644
index 0000000000..572a1b0da3
--- /dev/null
+++ b/backend/runner/taskrun/running_scheduler.go
@@ -0,0 +1,452 @@
+package taskrun
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/pkg/errors"
+ "google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/component/webhook"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+func getDatabaseKey(instanceID, databaseName string) string {
+ return fmt.Sprintf("%s/%s", instanceID, databaseName)
+}
+
+// runRunningTaskRunsScheduler runs in a separate goroutine to schedule running task runs.
+func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.WaitGroup) {
+ ticker := time.NewTicker(taskSchedulerInterval)
+ defer ticker.Stop()
+ defer wg.Done()
+ slog.Debug(fmt.Sprintf("Running task runs scheduler started and will run every %v", taskSchedulerInterval))
+ for {
+ select {
+ case <-ticker.C:
+ if err := s.scheduleRunningTaskRuns(ctx); err != nil {
+ slog.Error("failed to schedule running task runs", log.BBError(err))
+ }
+ case <-s.stateCfg.TaskRunTickleChan:
+ if err := s.scheduleRunningTaskRuns(ctx); err != nil {
+ slog.Error("failed to schedule running task runs", log.BBError(err))
+ }
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {
+ taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
+ Status: &[]storepb.TaskRun_Status{storepb.TaskRun_RUNNING},
+ })
+ if err != nil {
+ return errors.Wrapf(err, "failed to list pending tasks")
+ }
+
+ // Find the minimum task ID for each database.
+ // We only run the first (i.e. which has the minimum task ID) task for each database.
+ // 1. For ddl tasks, we run them one by one to get a sane schema dump and thus diff.
+ // 2. For versioned tasks, this is our last resort to determine the order for tasks with the same version. We don't want to run them in parallel.
+ // 2.1. Rollout 1 tasks will be run before rollout 2 tasks. Where, rollout 1 tasks are created before rollout 2 tasks.
+ minTaskIDForDatabase := map[string]int{}
+ for _, taskRun := range taskRuns {
+ task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
+ if err != nil {
+ slog.Error("failed to get task", slog.Int("task id", taskRun.TaskUID), log.BBError(err))
+ continue
+ }
+ if task.DatabaseName == nil {
+ continue
+ }
+
+ databaseKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)
+ if isSequentialTask(task) {
+ if _, ok := minTaskIDForDatabase[databaseKey]; !ok {
+ minTaskIDForDatabase[databaseKey] = task.ID
+ } else if minTaskIDForDatabase[databaseKey] > task.ID {
+ minTaskIDForDatabase[databaseKey] = task.ID
+ }
+ }
+ }
+
+ for _, taskRun := range taskRuns {
+ if err := s.scheduleRunningTaskRun(ctx, taskRun, minTaskIDForDatabase); err != nil {
+ slog.Error("failed to schedule running task run", log.BBError(err))
+ }
+ }
+
+ return nil
+}
+
+func (s *Scheduler) scheduleRunningTaskRun(ctx context.Context, taskRun *store.TaskRunMessage, minTaskIDForDatabase map[string]int) error {
+ // Skip the task run if it is already executing.
+ if _, ok := s.stateCfg.RunningTaskRuns.Load(taskRun.ID); ok {
+ return nil
+ }
+ task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get task")
+ }
+ if task.DatabaseName != nil && isSequentialTask(task) {
+ // Skip the task run if there is an ongoing migration on the database.
+ if taskUIDAny, ok := s.stateCfg.RunningDatabaseMigration.Load(getDatabaseKey(task.InstanceID, *task.DatabaseName)); ok {
+ if taskUID, ok := taskUIDAny.(int); ok {
+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
+ ReportTime: timestamppb.Now(),
+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{
+ Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
+ TaskUid: int32(taskUID),
+ },
+ },
+ })
+ }
+ return nil
+ }
+ if taskUID := minTaskIDForDatabase[getDatabaseKey(task.InstanceID, *task.DatabaseName)]; taskUID != task.ID {
+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
+ ReportTime: timestamppb.Now(),
+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{
+ Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
+ TaskUid: int32(taskUID),
+ },
+ },
+ })
+ return nil
+ }
+ }
+
+ instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})
+ if err != nil {
+ return errors.Wrapf(err, "failed to get instance")
+ }
+ if instance.Deleted {
+ return errors.Errorf("instance %v is deleted", task.InstanceID)
+ }
+ executor, ok := s.executorMap[task.Type]
+ if !ok {
+ return errors.Errorf("executor not found for task type: %v", task.Type)
+ }
+
+ // Check max connections per instance.
+ maximumConnections := int(instance.Metadata.GetMaximumConnections())
+ if maximumConnections <= 0 {
+ maximumConnections = common.DefaultInstanceMaximumConnections
+ }
+ if s.stateCfg.InstanceOutstandingConnections.Increment(task.InstanceID, maximumConnections) {
+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
+ ReportTime: timestamppb.Now(),
+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{
+ Cause: &storepb.SchedulerInfo_WaitingCause_ConnectionLimit{
+ ConnectionLimit: true,
+ },
+ },
+ })
+ return nil
+ }
+ // decrement the connection count if we return below.
+ revertInstanceConnectionsIncrement := true
+ defer func() {
+ if revertInstanceConnectionsIncrement {
+ s.stateCfg.InstanceOutstandingConnections.Decrement(task.InstanceID)
+ }
+ }()
+
+ // Check max running task runs per rollout.
+ pipeline, err := s.store.GetPipelineByID(ctx, task.PipelineID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get pipeline")
+ }
+ if pipeline == nil {
+ return errors.Errorf("pipeline %v not found", task.PipelineID)
+ }
+
+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &pipeline.ProjectID})
+ if err != nil {
+ return errors.Wrapf(err, "failed to get project")
+ }
+ if project == nil {
+ return errors.Errorf("project %v not found", pipeline.ProjectID)
+ }
+
+ rolloutID := strconv.Itoa(pipeline.ID)
+ maxRunningTaskRunsPerRollout := int(project.Setting.GetParallelTasksPerRollout())
+ if maxRunningTaskRunsPerRollout <= 0 {
+ maxRunningTaskRunsPerRollout = defaultRolloutMaxRunningTaskRuns
+ }
+ if s.stateCfg.RolloutOutstandingTasks.Increment(rolloutID+"/"+task.InstanceID, maxRunningTaskRunsPerRollout) {
+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
+ ReportTime: timestamppb.Now(),
+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{
+ Cause: &storepb.SchedulerInfo_WaitingCause_ParallelTasksLimit{
+ ParallelTasksLimit: true,
+ },
+ },
+ })
+ return nil
+ }
+
+ // decrement the connection count if we return below.
+ revertRolloutConnectionsIncrement := true
+ defer func() {
+ if revertRolloutConnectionsIncrement {
+ s.stateCfg.RolloutOutstandingTasks.Decrement(rolloutID + "/" + task.InstanceID)
+ }
+ }()
+
+ // Set taskrun StartAt when it's about to run.
+ // So that the waiting time is not taken into account of the actual execution time.
+ if err := s.store.UpdateTaskRunStartAt(ctx, taskRun.ID); err != nil {
+ return errors.Wrapf(err, "failed to update task run start at")
+ }
+
+ // We MUST NOT return early below this line.
+ // If we do want to return early, we must revert related states.
+ s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)
+ s.stateCfg.RunningTaskRuns.Store(taskRun.ID, true)
+ if task.DatabaseName != nil {
+ s.stateCfg.RunningDatabaseMigration.Store(getDatabaseKey(task.InstanceID, *task.DatabaseName), task.ID)
+ }
+
+ s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{
+ Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,
+ TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{
+ Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_RUNNING,
+ },
+ })
+
+ // We are sure that we will run the task.
+ // The executor will decrement them.
+ revertInstanceConnectionsIncrement = false
+ revertRolloutConnectionsIncrement = false
+ go s.runTaskRunOnce(ctx, taskRun, task, executor)
+ return nil
+}
+
+func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMessage, task *store.TaskMessage, executor Executor) {
+ defer func() {
+ if r := recover(); r != nil {
+ err, ok := r.(error)
+ if !ok {
+ err = errors.Errorf("%v", r)
+ }
+ slog.Error("Task scheduler V2 runTaskRunOnce PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
+ }
+ }()
+ defer func() {
+ // We don't need to do s.stateCfg.RunningTaskRuns.Delete(taskRun.ID) to avoid race condition.
+ s.stateCfg.RunningTaskRunsCancelFunc.Delete(taskRun.ID)
+ if task.DatabaseName != nil {
+ s.stateCfg.RunningDatabaseMigration.Delete(getDatabaseKey(task.InstanceID, *task.DatabaseName))
+ }
+ s.stateCfg.InstanceOutstandingConnections.Decrement(task.InstanceID)
+ s.stateCfg.RolloutOutstandingTasks.Decrement(strconv.Itoa(task.PipelineID) + "/" + task.InstanceID)
+ }()
+
+ driverCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ s.stateCfg.RunningTaskRunsCancelFunc.Store(taskRun.ID, cancel)
+
+ done, result, err := RunExecutorOnce(ctx, driverCtx, executor, task, taskRun.ID)
+
+ switch {
+ case !done && err != nil:
+ slog.Debug("Encountered transient error running task, will retry",
+ slog.Int("id", task.ID),
+ slog.String("type", task.Type.String()),
+ log.BBError(err),
+ )
+ return
+
+ case done && err != nil && errors.Is(err, context.Canceled):
+ slog.Warn("task run is canceled",
+ slog.Int("id", task.ID),
+ slog.String("type", task.Type.String()),
+ log.BBError(err),
+ )
+ resultBytes, marshalErr := protojson.Marshal(&storepb.TaskRunResult{
+ Detail: "The task run is canceled",
+ Changelog: "",
+ Version: "",
+ })
+ if marshalErr != nil {
+ slog.Error("Failed to marshal task run result",
+ slog.Int("task_id", task.ID),
+ slog.String("type", task.Type.String()),
+ log.BBError(marshalErr),
+ )
+ return
+ }
+ code := common.Ok
+ result := string(resultBytes)
+ taskRunStatusPatch := &store.TaskRunStatusPatch{
+ ID: taskRun.ID,
+ Updater: common.SystemBotEmail,
+ Status: storepb.TaskRun_CANCELED,
+ Code: &code,
+ Result: &result,
+ }
+ if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {
+ slog.Error("Failed to mark task as CANCELED",
+ slog.Int("id", task.ID),
+ log.BBError(err),
+ )
+ return
+ }
+ return
+
+ case done && err != nil:
+ slog.Warn("task run failed",
+ slog.Int("id", task.ID),
+ slog.String("type", task.Type.String()),
+ log.BBError(err),
+ )
+ taskRunResult := &storepb.TaskRunResult{
+ Detail: err.Error(),
+ Changelog: "",
+ Version: "",
+ }
+ var errWithPosition *db.ErrorWithPosition
+ if errors.As(err, &errWithPosition) {
+ taskRunResult.StartPosition = errWithPosition.Start
+ taskRunResult.EndPosition = errWithPosition.End
+ }
+ resultBytes, marshalErr := protojson.Marshal(taskRunResult)
+ if marshalErr != nil {
+ slog.Error("Failed to marshal task run result",
+ slog.Int("task_id", task.ID),
+ slog.String("type", task.Type.String()),
+ log.BBError(marshalErr),
+ )
+ return
+ }
+ code := common.ErrorCode(err)
+ result := string(resultBytes)
+ taskRunStatusPatch := &store.TaskRunStatusPatch{
+ ID: taskRun.ID,
+ Updater: common.SystemBotEmail,
+ Status: storepb.TaskRun_FAILED,
+ Code: &code,
+ Result: &result,
+ }
+ if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {
+ slog.Error("Failed to mark task as FAILED",
+ slog.Int("id", task.ID),
+ log.BBError(err),
+ )
+ return
+ }
+ s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_FAILED, taskRunResult.Detail)
+ return
+
+ case done && err == nil:
+ resultBytes, marshalErr := protojson.Marshal(result)
+ if marshalErr != nil {
+ slog.Error("Failed to marshal task run result",
+ slog.Int("task_id", task.ID),
+ slog.String("type", task.Type.String()),
+ log.BBError(marshalErr),
+ )
+ return
+ }
+ code := common.Ok
+ result := string(resultBytes)
+ taskRunStatusPatch := &store.TaskRunStatusPatch{
+ ID: taskRun.ID,
+ Updater: common.SystemBotEmail,
+ Status: storepb.TaskRun_DONE,
+ Code: &code,
+ Result: &result,
+ }
+ if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {
+ slog.Error("Failed to mark task as DONE",
+ slog.Int("id", task.ID),
+ log.BBError(err),
+ )
+ return
+ }
+ s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_DONE, "")
+ s.stateCfg.TaskSkippedOrDoneChan <- task.ID
+ return
+ default:
+ // This case should not happen in normal flow, but adding for completeness
+ slog.Error("Unexpected task execution state",
+ slog.Int("id", task.ID),
+ slog.String("type", task.Type.String()),
+ slog.Bool("done", done),
+ slog.Bool("has_error", err != nil),
+ )
+ return
+ }
+}
+
+func (s *Scheduler) createActivityForTaskRunStatusUpdate(ctx context.Context, task *store.TaskMessage, newStatus storepb.TaskRun_Status, errDetail string) {
+ if err := func() error {
+ rollout, err := s.store.GetPipelineByID(ctx, task.PipelineID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get pipeline")
+ }
+ if rollout == nil {
+ return errors.Errorf("pipeline %v not found", task.PipelineID)
+ }
+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &rollout.ProjectID})
+ if err != nil {
+ return errors.Wrapf(err, "failed to get project")
+ }
+ if project == nil {
+ return errors.Errorf("project %v not found", rollout.ProjectID)
+ }
+ issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{
+ PipelineID: &task.PipelineID,
+ })
+ if err != nil {
+ return errors.Wrap(err, "failed to get issue")
+ }
+ s.webhookManager.CreateEvent(ctx, &webhook.Event{
+ Actor: store.SystemBotUser,
+ Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,
+ Comment: "",
+ Issue: webhook.NewIssue(issue),
+ Rollout: webhook.NewRollout(rollout),
+ Project: webhook.NewProject(project),
+ TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{
+ Title: task.GetDatabaseName(),
+ Status: newStatus.String(),
+ Detail: errDetail,
+ },
+ })
+ return nil
+ }(); err != nil {
+ slog.Error("failed to create activity for task run status update", log.BBError(err))
+ }
+}
+
+// isSequentialTask returns whether the task should be executed sequentially.
+func isSequentialTask(task *store.TaskMessage) bool {
+ //exhaustive:enforce
+ switch task.Type {
+ case storepb.Task_DATABASE_MIGRATE:
+ // All DATABASE_MIGRATE operations (DDL/GHOST) should be sequential
+ return true
+ case storepb.Task_DATABASE_SDL:
+ // SDL operations should be sequential
+ return true
+ case storepb.Task_DATABASE_CREATE,
+ storepb.Task_DATABASE_EXPORT:
+ return false
+ case storepb.Task_TASK_TYPE_UNSPECIFIED:
+ return false
+ default:
+ return false
+ }
+}
diff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.go
index ed5ebb8fb2..af0c5d78e6 100644
--- a/backend/runner/taskrun/scheduler.go
+++ b/backend/runner/taskrun/scheduler.go
@@ -2,15 +2,11 @@ package taskrun
import (
"context"
- "fmt"
"log/slog"
- "strconv"
"sync"
"time"
"github.com/pkg/errors"
- "google.golang.org/protobuf/encoding/protojson"
- "google.golang.org/protobuf/types/known/timestamppb"
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
@@ -19,9 +15,7 @@ import (
"github.com/bytebase/bytebase/backend/component/webhook"
"github.com/bytebase/bytebase/backend/enterprise"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/store"
- "github.com/bytebase/bytebase/backend/utils"
)
const (
@@ -73,621 +67,18 @@ func (s *Scheduler) Register(taskType storepb.Task_Type, executorGetter Executor
// Run will start the scheduler.
func (s *Scheduler) Run(ctx context.Context, wg *sync.WaitGroup) {
- go s.ListenTaskSkippedOrDone(ctx)
-
- ticker := time.NewTicker(taskSchedulerInterval)
- defer ticker.Stop()
defer wg.Done()
- slog.Debug(fmt.Sprintf("Task scheduler V2 started and will run every %v", taskSchedulerInterval))
- for {
- select {
- case <-ticker.C:
- s.runOnce(ctx)
- case <-s.stateCfg.TaskRunTickleChan:
- s.runOnce(ctx)
- case <-ctx.Done():
- return
- }
- }
-}
-
-func (s *Scheduler) runOnce(ctx context.Context) {
- defer func() {
- if r := recover(); r != nil {
- err, ok := r.(error)
- if !ok {
- err = errors.Errorf("%v", r)
- }
- slog.Error("Task scheduler V2 PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
- }
- }()
-
- if err := s.scheduleAutoRolloutTasks(ctx); err != nil {
- slog.Error("failed to schedule auto rollout tasks", log.BBError(err))
- }
-
- if err := s.schedulePendingTaskRuns(ctx); err != nil {
- slog.Error("failed to schedule pending task runs", log.BBError(err))
- }
-
- if err := s.scheduleRunningTaskRuns(ctx); err != nil {
- slog.Error("failed to schedule running task runs", log.BBError(err))
- }
-}
-
-func (s *Scheduler) scheduleAutoRolloutTasks(ctx context.Context) error {
- environments, err := s.store.GetEnvironment(ctx)
- if err != nil {
- return errors.Wrapf(err, "failed to list environments")
- }
-
- var envs []string
- for _, environment := range environments.GetEnvironments() {
- policy, err := s.store.GetRolloutPolicy(ctx, environment.Id)
- if err != nil {
- return errors.Wrapf(err, "failed to get rollout policy for environment %s", environment.Id)
- }
- if policy.Automatic {
- envs = append(envs, environment.Id)
- }
- }
- taskIDs, err := s.store.ListTasksToAutoRollout(ctx, envs)
- if err != nil {
- return errors.Wrapf(err, "failed to list tasks with zero task run")
- }
- for _, taskID := range taskIDs {
- if err := s.scheduleAutoRolloutTask(ctx, taskID); err != nil {
- slog.Error("failed to schedule auto rollout task", log.BBError(err))
- }
- }
-
- return nil
-}
-
-func (s *Scheduler) scheduleAutoRolloutTask(ctx context.Context, taskUID int) error {
- task, err := s.store.GetTaskByID(ctx, taskUID)
- if err != nil {
- return errors.Wrapf(err, "failed to get task")
- }
- if task == nil {
- return nil
- }
-
- pipeline, err := s.store.GetPipelineByID(ctx, task.PipelineID)
- if err != nil {
- return errors.Wrapf(err, "failed to get pipeline")
- }
- if pipeline == nil {
- return errors.Errorf("pipeline %v not found", task.PipelineID)
- }
-
- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &pipeline.ProjectID})
- if err != nil {
- return errors.Wrapf(err, "failed to get project")
- }
- if project == nil {
- return errors.Errorf("project %v not found", pipeline.ProjectID)
- }
-
- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})
- if err != nil {
- return err
- }
- if instance.Deleted {
- return nil
- }
-
- issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PipelineID: &task.PipelineID})
- if err != nil {
- return err
- }
- if issue != nil {
- if issue.Status != storepb.Issue_OPEN {
- return nil
- }
- // Check if issue approval is required according to the project settings
- if project.Setting.RequireIssueApproval {
- approved, err := utils.CheckIssueApproved(issue)
- if err != nil {
- return errors.Wrapf(err, "failed to check if the issue is approved")
- }
- if !approved {
- return nil
- }
- }
- }
-
- // Check the latest plan checks based on project settings (error only)
- if project.Setting.RequirePlanCheckNoError {
- pass, err := func() (bool, error) {
- plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{PipelineID: &task.PipelineID})
- if err != nil {
- return false, errors.Wrapf(err, "failed to get plan")
- }
- if plan == nil {
- return true, nil
- }
- latestRuns, err := s.store.ListPlanCheckRuns(ctx, &store.FindPlanCheckRunMessage{
- PlanUID: &plan.UID,
- })
- if err != nil {
- return false, errors.Wrapf(err, "failed to list latest plan check runs")
- }
- for _, run := range latestRuns {
- if run.Status != store.PlanCheckRunStatusDone {
- return false, nil
- }
- for _, result := range run.Result.Results {
- if result.Status == storepb.Advice_ERROR {
- return false, nil
- }
- }
- }
- return true, nil
- }()
- if err != nil {
- return errors.Wrapf(err, "failed to check if plan check passes")
- }
- if !pass {
- return nil
- }
- }
-
- create := &store.TaskRunMessage{
- TaskUID: task.ID,
- }
- if task.Payload.GetSheetId() != 0 {
- sheetUID := int(task.Payload.GetSheetId())
- create.SheetUID = &sheetUID
- }
-
- if err := s.store.CreatePendingTaskRuns(ctx, common.SystemBotEmail, create); err != nil {
- return errors.Wrapf(err, "failed to create pending task runs")
- }
-
- s.webhookManager.CreateEvent(ctx, &webhook.Event{
- Actor: store.SystemBotUser,
- Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,
- Comment: "",
- Issue: webhook.NewIssue(issue),
- Rollout: webhook.NewRollout(pipeline),
- Project: webhook.NewProject(project),
- TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{
- Title: task.GetDatabaseName(),
- Status: storepb.TaskRun_PENDING.String(),
- },
- })
-
- return nil
-}
-
-func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
- taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_PENDING},
- })
- if err != nil {
- return errors.Wrapf(err, "failed to list pending tasks")
- }
- for _, taskRun := range taskRuns {
- if err := s.schedulePendingTaskRun(ctx, taskRun); err != nil {
- slog.Error("failed to schedule pending task run", log.BBError(err))
- }
- }
-
- return nil
-}
-
-func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {
- // here, we move pending taskruns to running taskruns which means they are ready to be executed.
- // pending taskruns remain pending if
- // 1. taskRun.RunAt not met.
- // 2. for versioned tasks, there are other versioned tasks on the same database with
- // a smaller version not finished yet. we need to wait for those first.
- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
- if err != nil {
- return errors.Wrapf(err, "failed to get task")
- }
- if taskRun.RunAt != nil && time.Now().Before(*taskRun.RunAt) {
- return nil
- }
-
- doSchedule, err := func() (bool, error) {
- if task.DatabaseName == nil {
- return true, nil
- }
-
- schemaVersion := task.Payload.GetSchemaVersion()
- if schemaVersion == "" {
- return true, nil
- }
-
- maybeTaskID, err := s.store.FindBlockingTaskByVersion(ctx, task.PipelineID, task.InstanceID, *task.DatabaseName, schemaVersion)
- if err != nil {
- return false, errors.Wrapf(err, "failed to find blocking versioned tasks")
- }
- if maybeTaskID != nil {
- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
- ReportTime: timestamppb.Now(),
- WaitingCause: &storepb.SchedulerInfo_WaitingCause{
- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
- TaskUid: int32(*maybeTaskID),
- },
- },
- })
- return false, nil
- }
- s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)
- return true, nil
- }()
- if err != nil {
- return errors.Wrapf(err, "failed to check blocking versioned tasks")
- }
- if !doSchedule {
- return nil
- }
- if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{
- ID: taskRun.ID,
- Updater: common.SystemBotEmail,
- Status: storepb.TaskRun_RUNNING,
- }); err != nil {
- return errors.Wrapf(err, "failed to update task run status to running")
- }
- s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{
- Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,
- TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{
- Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_WAITING,
- },
- })
- return nil
-}
-
-func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {
- taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_RUNNING},
- })
- if err != nil {
- return errors.Wrapf(err, "failed to list pending tasks")
- }
-
- // Find the minimum task ID for each database.
- // We only run the first (i.e. which has the minimum task ID) task for each database.
- // 1. For ddl tasks, we run them one by one to get a sane schema dump and thus diff.
- // 2. For versioned tasks, this is our last resort to determine the order for tasks with the same version. We don't want to run them in parallel.
- // 2.1. Rollout 1 tasks will be run before rollout 2 tasks. Where, rollout 1 tasks are created before rollout 2 tasks.
- minTaskIDForDatabase := map[string]int{}
- for _, taskRun := range taskRuns {
- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
- if err != nil {
- slog.Error("failed to get task", slog.Int("task id", taskRun.TaskUID), log.BBError(err))
- continue
- }
- if task.DatabaseName == nil {
- continue
- }
-
- databaseKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)
- if isSequentialTask(task) {
- if _, ok := minTaskIDForDatabase[databaseKey]; !ok {
- minTaskIDForDatabase[databaseKey] = task.ID
- } else if minTaskIDForDatabase[databaseKey] > task.ID {
- minTaskIDForDatabase[databaseKey] = task.ID
- }
- }
- }
-
- for _, taskRun := range taskRuns {
- if err := s.scheduleRunningTaskRun(ctx, taskRun, minTaskIDForDatabase); err != nil {
- slog.Error("failed to schedule running task run", log.BBError(err))
- }
- }
-
- return nil
-}
-
-func (s *Scheduler) scheduleRunningTaskRun(ctx context.Context, taskRun *store.TaskRunMessage, minTaskIDForDatabase map[string]int) error {
- // Skip the task run if it is already executing.
- if _, ok := s.stateCfg.RunningTaskRuns.Load(taskRun.ID); ok {
- return nil
- }
- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
- if err != nil {
- return errors.Wrapf(err, "failed to get task")
- }
- if task.DatabaseName != nil && isSequentialTask(task) {
- // Skip the task run if there is an ongoing migration on the database.
- if taskUIDAny, ok := s.stateCfg.RunningDatabaseMigration.Load(getDatabaseKey(task.InstanceID, *task.DatabaseName)); ok {
- if taskUID, ok := taskUIDAny.(int); ok {
- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
- ReportTime: timestamppb.Now(),
- WaitingCause: &storepb.SchedulerInfo_WaitingCause{
- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
- TaskUid: int32(taskUID),
- },
- },
- })
- }
- return nil
- }
- if taskUID := minTaskIDForDatabase[getDatabaseKey(task.InstanceID, *task.DatabaseName)]; taskUID != task.ID {
- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
- ReportTime: timestamppb.Now(),
- WaitingCause: &storepb.SchedulerInfo_WaitingCause{
- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
- TaskUid: int32(taskUID),
- },
- },
- })
- return nil
- }
- }
-
- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})
- if err != nil {
- return errors.Wrapf(err, "failed to get instance")
- }
- if instance.Deleted {
- return errors.Errorf("instance %v is deleted", task.InstanceID)
- }
- executor, ok := s.executorMap[task.Type]
- if !ok {
- return errors.Errorf("executor not found for task type: %v", task.Type)
- }
-
- // Check max connections per instance.
- maximumConnections := int(instance.Metadata.GetMaximumConnections())
- if maximumConnections <= 0 {
- maximumConnections = common.DefaultInstanceMaximumConnections
- }
- if s.stateCfg.InstanceOutstandingConnections.Increment(task.InstanceID, maximumConnections) {
- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
- ReportTime: timestamppb.Now(),
- WaitingCause: &storepb.SchedulerInfo_WaitingCause{
- Cause: &storepb.SchedulerInfo_WaitingCause_ConnectionLimit{
- ConnectionLimit: true,
- },
- },
- })
- return nil
- }
- // decrement the connection count if we return below.
- revertInstanceConnectionsIncrement := true
- defer func() {
- if revertInstanceConnectionsIncrement {
- s.stateCfg.InstanceOutstandingConnections.Decrement(task.InstanceID)
- }
- }()
-
- // Check max running task runs per rollout.
- pipeline, err := s.store.GetPipelineByID(ctx, task.PipelineID)
- if err != nil {
- return errors.Wrapf(err, "failed to get pipeline")
- }
- if pipeline == nil {
- return errors.Errorf("pipeline %v not found", task.PipelineID)
- }
-
- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &pipeline.ProjectID})
- if err != nil {
- return errors.Wrapf(err, "failed to get project")
- }
- if project == nil {
- return errors.Errorf("project %v not found", pipeline.ProjectID)
- }
-
- rolloutID := strconv.Itoa(pipeline.ID)
- maxRunningTaskRunsPerRollout := int(project.Setting.GetParallelTasksPerRollout())
- if maxRunningTaskRunsPerRollout <= 0 {
- maxRunningTaskRunsPerRollout = defaultRolloutMaxRunningTaskRuns
- }
- if s.stateCfg.RolloutOutstandingTasks.Increment(rolloutID+"/"+task.InstanceID, maxRunningTaskRunsPerRollout) {
- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
- ReportTime: timestamppb.Now(),
- WaitingCause: &storepb.SchedulerInfo_WaitingCause{
- Cause: &storepb.SchedulerInfo_WaitingCause_ParallelTasksLimit{
- ParallelTasksLimit: true,
- },
- },
- })
- return nil
- }
-
- // decrement the connection count if we return below.
- revertRolloutConnectionsIncrement := true
- defer func() {
- if revertRolloutConnectionsIncrement {
- s.stateCfg.RolloutOutstandingTasks.Decrement(rolloutID + "/" + task.InstanceID)
- }
- }()
-
- // Set taskrun StartAt when it's about to run.
- // So that the waiting time is not taken into account of the actual execution time.
- if err := s.store.UpdateTaskRunStartAt(ctx, taskRun.ID); err != nil {
- return errors.Wrapf(err, "failed to update task run start at")
- }
-
- // We MUST NOT return early below this line.
- // If we do want to return early, we must revert related states.
- s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)
- s.stateCfg.RunningTaskRuns.Store(taskRun.ID, true)
- if task.DatabaseName != nil {
- s.stateCfg.RunningDatabaseMigration.Store(getDatabaseKey(task.InstanceID, *task.DatabaseName), task.ID)
- }
-
- s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{
- Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,
- TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{
- Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_RUNNING,
- },
- })
-
- // We are sure that we will run the task.
- // The executor will decrement them.
- revertInstanceConnectionsIncrement = false
- revertRolloutConnectionsIncrement = false
- go s.runTaskRunOnce(ctx, taskRun, task, executor)
- return nil
-}
-
-func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMessage, task *store.TaskMessage, executor Executor) {
- defer func() {
- if r := recover(); r != nil {
- err, ok := r.(error)
- if !ok {
- err = errors.Errorf("%v", r)
- }
- slog.Error("Task scheduler V2 runTaskRunOnce PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
- }
- }()
- defer func() {
- // We don't need to do s.stateCfg.RunningTaskRuns.Delete(taskRun.ID) to avoid race condition.
- s.stateCfg.RunningTaskRunsCancelFunc.Delete(taskRun.ID)
- if task.DatabaseName != nil {
- s.stateCfg.RunningDatabaseMigration.Delete(getDatabaseKey(task.InstanceID, *task.DatabaseName))
- }
- s.stateCfg.InstanceOutstandingConnections.Decrement(task.InstanceID)
- s.stateCfg.RolloutOutstandingTasks.Decrement(strconv.Itoa(task.PipelineID) + "/" + task.InstanceID)
- }()
+ go s.ListenTaskSkippedOrDone(ctx)
- driverCtx, cancel := context.WithCancel(ctx)
- defer cancel()
- s.stateCfg.RunningTaskRunsCancelFunc.Store(taskRun.ID, cancel)
-
- done, result, err := RunExecutorOnce(ctx, driverCtx, executor, task, taskRun.ID)
-
- switch {
- case !done && err != nil:
- slog.Debug("Encountered transient error running task, will retry",
- slog.Int("id", task.ID),
- slog.String("type", task.Type.String()),
- log.BBError(err),
- )
- return
-
- case done && err != nil && errors.Is(err, context.Canceled):
- slog.Warn("task run is canceled",
- slog.Int("id", task.ID),
- slog.String("type", task.Type.String()),
- log.BBError(err),
- )
- resultBytes, marshalErr := protojson.Marshal(&storepb.TaskRunResult{
- Detail: "The task run is canceled",
- Changelog: "",
- Version: "",
- })
- if marshalErr != nil {
- slog.Error("Failed to marshal task run result",
- slog.Int("task_id", task.ID),
- slog.String("type", task.Type.String()),
- log.BBError(marshalErr),
- )
- return
- }
- code := common.Ok
- result := string(resultBytes)
- taskRunStatusPatch := &store.TaskRunStatusPatch{
- ID: taskRun.ID,
- Updater: common.SystemBotEmail,
- Status: storepb.TaskRun_CANCELED,
- Code: &code,
- Result: &result,
- }
- if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {
- slog.Error("Failed to mark task as CANCELED",
- slog.Int("id", task.ID),
- log.BBError(err),
- )
- return
- }
- return
-
- case done && err != nil:
- slog.Warn("task run failed",
- slog.Int("id", task.ID),
- slog.String("type", task.Type.String()),
- log.BBError(err),
- )
- taskRunResult := &storepb.TaskRunResult{
- Detail: err.Error(),
- Changelog: "",
- Version: "",
- }
- var errWithPosition *db.ErrorWithPosition
- if errors.As(err, &errWithPosition) {
- taskRunResult.StartPosition = errWithPosition.Start
- taskRunResult.EndPosition = errWithPosition.End
- }
- resultBytes, marshalErr := protojson.Marshal(taskRunResult)
- if marshalErr != nil {
- slog.Error("Failed to marshal task run result",
- slog.Int("task_id", task.ID),
- slog.String("type", task.Type.String()),
- log.BBError(marshalErr),
- )
- return
- }
- code := common.ErrorCode(err)
- result := string(resultBytes)
- taskRunStatusPatch := &store.TaskRunStatusPatch{
- ID: taskRun.ID,
- Updater: common.SystemBotEmail,
- Status: storepb.TaskRun_FAILED,
- Code: &code,
- Result: &result,
- }
- if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {
- slog.Error("Failed to mark task as FAILED",
- slog.Int("id", task.ID),
- log.BBError(err),
- )
- return
- }
- s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_FAILED, taskRunResult.Detail)
- return
-
- case done && err == nil:
- resultBytes, marshalErr := protojson.Marshal(result)
- if marshalErr != nil {
- slog.Error("Failed to marshal task run result",
- slog.Int("task_id", task.ID),
- slog.String("type", task.Type.String()),
- log.BBError(marshalErr),
- )
- return
- }
- code := common.Ok
- result := string(resultBytes)
- taskRunStatusPatch := &store.TaskRunStatusPatch{
- ID: taskRun.ID,
- Updater: common.SystemBotEmail,
- Status: storepb.TaskRun_DONE,
- Code: &code,
- Result: &result,
- }
- if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {
- slog.Error("Failed to mark task as DONE",
- slog.Int("id", task.ID),
- log.BBError(err),
- )
- return
- }
- s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_DONE, "")
- s.stateCfg.TaskSkippedOrDoneChan <- task.ID
- return
- default:
- // This case should not happen in normal flow, but adding for completeness
- slog.Error("Unexpected task execution state",
- slog.Int("id", task.ID),
- slog.String("type", task.Type.String()),
- slog.Bool("done", done),
- slog.Bool("has_error", err != nil),
- )
- return
- }
-}
+ // Start three independent schedulers
+ wg.Add(3)
+ go s.runAutoRolloutScheduler(ctx, wg)
+ go s.runPendingTaskRunsScheduler(ctx, wg)
+ go s.runRunningTaskRunsScheduler(ctx, wg)
-func getDatabaseKey(instanceID, databaseName string) string {
- return fmt.Sprintf("%s/%s", instanceID, databaseName)
+ slog.Debug("Task scheduler V2 started with independent runners")
+ <-ctx.Done()
}
func (s *Scheduler) ListenTaskSkippedOrDone(ctx context.Context) {
@@ -893,47 +284,6 @@ func (s *Scheduler) ListenTaskSkippedOrDone(ctx context.Context) {
}
}
-func (s *Scheduler) createActivityForTaskRunStatusUpdate(ctx context.Context, task *store.TaskMessage, newStatus storepb.TaskRun_Status, errDetail string) {
- if err := func() error {
- rollout, err := s.store.GetPipelineByID(ctx, task.PipelineID)
- if err != nil {
- return errors.Wrapf(err, "failed to get pipeline")
- }
- if rollout == nil {
- return errors.Errorf("pipeline %v not found", task.PipelineID)
- }
- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &rollout.ProjectID})
- if err != nil {
- return errors.Wrapf(err, "failed to get project")
- }
- if project == nil {
- return errors.Errorf("project %v not found", rollout.ProjectID)
- }
- issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{
- PipelineID: &task.PipelineID,
- })
- if err != nil {
- return errors.Wrap(err, "failed to get issue")
- }
- s.webhookManager.CreateEvent(ctx, &webhook.Event{
- Actor: store.SystemBotUser,
- Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,
- Comment: "",
- Issue: webhook.NewIssue(issue),
- Rollout: webhook.NewRollout(rollout),
- Project: webhook.NewProject(project),
- TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{
- Title: task.GetDatabaseName(),
- Status: newStatus.String(),
- Detail: errDetail,
- },
- })
- return nil
- }(); err != nil {
- slog.Error("failed to create activity for task run status update", log.BBError(err))
- }
-}
-
func tasksSkippedOrDone(tasks []*store.TaskMessage) bool {
for _, task := range tasks {
skipped := task.Payload.GetSkipped()
@@ -945,26 +295,6 @@ func tasksSkippedOrDone(tasks []*store.TaskMessage) bool {
return true
}
-// isSequentialTask returns whether the task should be executed sequentially.
-func isSequentialTask(task *store.TaskMessage) bool {
- //exhaustive:enforce
- switch task.Type {
- case storepb.Task_DATABASE_MIGRATE:
- // All DATABASE_MIGRATE operations (DDL/GHOST) should be sequential
- return true
- case storepb.Task_DATABASE_SDL:
- // SDL operations should be sequential
- return true
- case storepb.Task_DATABASE_CREATE,
- storepb.Task_DATABASE_EXPORT:
- return false
- case storepb.Task_TASK_TYPE_UNSPECIFIED:
- return false
- default:
- return false
- }
-}
-
// getAllEnvironmentIDs returns all environment IDs from the store.
func getAllEnvironmentIDs(ctx context.Context, s *store.Store) ([]string, error) {
environments, err := s.GetEnvironment(ctx)