Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: split task scheduler into independent schedulers (#18517)
Danny Xu47e8f8e52544
diff --git a/backend/runner/taskrun/auto_rollout_scheduler.go b/backend/runner/taskrun/auto_rollout_scheduler.gonew file mode 100644index 0000000000..d85ec24d99--- /dev/null+++ b/backend/runner/taskrun/auto_rollout_scheduler.go@@ -0,0 +1,182 @@+package taskrun++import (+ "context"+ "log/slog"+ "sync"+ "time"++ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log"+ "github.com/bytebase/bytebase/backend/component/webhook"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/store"+ "github.com/bytebase/bytebase/backend/utils"+)++// runAutoRolloutScheduler runs in a separate goroutine to schedule auto-rollout tasks.+// This prevents blocking the main scheduler when there are many tasks to auto-rollout.+func (s *Scheduler) runAutoRolloutScheduler(ctx context.Context, wg *sync.WaitGroup) {+ ticker := time.NewTicker(10 * time.Second)+ defer ticker.Stop()+ defer wg.Done()+ slog.Debug("Auto-rollout scheduler started and will run every 10s")+ for {+ select {+ case <-ticker.C:+ if err := s.scheduleAutoRolloutTasks(ctx); err != nil {+ slog.Error("failed to schedule auto rollout tasks", log.BBError(err))+ }+ case <-ctx.Done():+ return+ }+ }+}++func (s *Scheduler) scheduleAutoRolloutTasks(ctx context.Context) error {+ environments, err := s.store.GetEnvironment(ctx)+ if err != nil {+ return errors.Wrapf(err, "failed to list environments")+ }++ var envs []string+ for _, environment := range environments.GetEnvironments() {+ policy, err := s.store.GetRolloutPolicy(ctx, environment.Id)+ if err != nil {+ return errors.Wrapf(err, "failed to get rollout policy for environment %s", environment.Id)+ }+ if policy.Automatic {+ envs = append(envs, environment.Id)+ }+ }+ taskIDs, err := s.store.ListTasksToAutoRollout(ctx, envs)+ if err != nil {+ return errors.Wrapf(err, "failed to list tasks with zero task run")+ }+ for _, taskID := range taskIDs {+ if err := s.scheduleAutoRolloutTask(ctx, taskID); err != nil {+ slog.Error("failed to schedule auto rollout task", log.BBError(err))+ }+ }++ return nil+}++func (s *Scheduler) scheduleAutoRolloutTask(ctx context.Context, taskUID int) error {+ task, err := s.store.GetTaskByID(ctx, taskUID)+ if err != nil {+ return errors.Wrapf(err, "failed to get task")+ }+ if task == nil {+ return nil+ }++ pipeline, err := s.store.GetPipelineByID(ctx, task.PipelineID)+ if err != nil {+ return errors.Wrapf(err, "failed to get pipeline")+ }+ if pipeline == nil {+ return errors.Errorf("pipeline %v not found", task.PipelineID)+ }++ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &pipeline.ProjectID})+ if err != nil {+ return errors.Wrapf(err, "failed to get project")+ }+ if project == nil {+ return errors.Errorf("project %v not found", pipeline.ProjectID)+ }++ instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})+ if err != nil {+ return err+ }+ if instance.Deleted {+ return nil+ }++ issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PipelineID: &task.PipelineID})+ if err != nil {+ return err+ }+ if issue != nil {+ if issue.Status != storepb.Issue_OPEN {+ return nil+ }+ // Check if issue approval is required according to the project settings+ if project.Setting.RequireIssueApproval {+ approved, err := utils.CheckIssueApproved(issue)+ if err != nil {+ return errors.Wrapf(err, "failed to check if the issue is approved")+ }+ if !approved {+ return nil+ }+ }+ }++ // Check the latest plan checks based on project settings (error only)+ if project.Setting.RequirePlanCheckNoError {+ pass, err := func() (bool, error) {+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{PipelineID: &task.PipelineID})+ if err != nil {+ return false, errors.Wrapf(err, "failed to get plan")+ }+ if plan == nil {+ return true, nil+ }+ latestRuns, err := s.store.ListPlanCheckRuns(ctx, &store.FindPlanCheckRunMessage{+ PlanUID: &plan.UID,+ })+ if err != nil {+ return false, errors.Wrapf(err, "failed to list latest plan check runs")+ }+ for _, run := range latestRuns {+ if run.Status != store.PlanCheckRunStatusDone {+ return false, nil+ }+ for _, result := range run.Result.Results {+ if result.Status == storepb.Advice_ERROR {+ return false, nil+ }+ }+ }+ return true, nil+ }()+ if err != nil {+ return errors.Wrapf(err, "failed to check if plan check passes")+ }+ if !pass {+ return nil+ }+ }++ create := &store.TaskRunMessage{+ TaskUID: task.ID,+ }+ if task.Payload.GetSheetId() != 0 {+ sheetUID := int(task.Payload.GetSheetId())+ create.SheetUID = &sheetUID+ }++ if err := s.store.CreatePendingTaskRuns(ctx, common.SystemBotEmail, create); err != nil {+ return errors.Wrapf(err, "failed to create pending task runs")+ }++ s.webhookManager.CreateEvent(ctx, &webhook.Event{+ Actor: store.SystemBotUser,+ Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,+ Comment: "",+ Issue: webhook.NewIssue(issue),+ Rollout: webhook.NewRollout(pipeline),+ Project: webhook.NewProject(project),+ TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{+ Title: task.GetDatabaseName(),+ Status: storepb.TaskRun_PENDING.String(),+ },+ })++ return nil+}diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.gonew file mode 100644index 0000000000..a1253308e3--- /dev/null+++ b/backend/runner/taskrun/pending_scheduler.go@@ -0,0 +1,129 @@+package taskrun++import (+ "context"+ "fmt"+ "log/slog"+ "sync"+ "time"++ "github.com/pkg/errors"+ "google.golang.org/protobuf/types/known/timestamppb"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/store"+)++// runPendingTaskRunsScheduler runs in a separate goroutine to schedule pending task runs.+func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.WaitGroup) {+ ticker := time.NewTicker(taskSchedulerInterval)+ defer ticker.Stop()+ defer wg.Done()+ slog.Debug(fmt.Sprintf("Pending task runs scheduler started and will run every %v", taskSchedulerInterval))+ for {+ select {+ case <-ticker.C:+ if err := s.schedulePendingTaskRuns(ctx); err != nil {+ slog.Error("failed to schedule pending task runs", log.BBError(err))+ }+ case <-s.stateCfg.TaskRunTickleChan:+ if err := s.schedulePendingTaskRuns(ctx); err != nil {+ slog.Error("failed to schedule pending task runs", log.BBError(err))+ }+ case <-ctx.Done():+ return+ }+ }+}++func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {+ taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{+ Status: &[]storepb.TaskRun_Status{storepb.TaskRun_PENDING},+ })+ if err != nil {+ return errors.Wrapf(err, "failed to list pending tasks")+ }+ for _, taskRun := range taskRuns {+ if err := s.schedulePendingTaskRun(ctx, taskRun); err != nil {+ slog.Error("failed to schedule pending task run", log.BBError(err))+ }+ }++ return nil+}++func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {+ // here, we move pending taskruns to running taskruns which means they are ready to be executed.+ // pending taskruns remain pending if+ // 1. taskRun.RunAt not met.+ // 2. for versioned tasks, there are other versioned tasks on the same database with+ // a smaller version not finished yet. we need to wait for those first.+ task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)+ if err != nil {+ return errors.Wrapf(err, "failed to get task")+ }+ if taskRun.RunAt != nil && time.Now().Before(*taskRun.RunAt) {+ return nil+ }++ doSchedule, err := func() (bool, error) {+ if task.DatabaseName == nil {+ return true, nil+ }++ schemaVersion := task.Payload.GetSchemaVersion()+ if schemaVersion == "" {+ return true, nil+ }++ maybeTaskID, err := s.store.FindBlockingTaskByVersion(ctx, task.PipelineID, task.InstanceID, *task.DatabaseName, schemaVersion)+ if err != nil {+ return false, errors.Wrapf(err, "failed to find blocking versioned tasks")+ }+ if maybeTaskID != nil {+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{+ ReportTime: timestamppb.Now(),+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{+ Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{+ TaskUid: int32(*maybeTaskID),+ },+ },+ })+ return false, nil+ }+ s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)+ return true, nil+ }()+ if err != nil {+ return errors.Wrapf(err, "failed to check blocking versioned tasks")+ }+ if !doSchedule {+ return nil+ }++ if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{+ ID: taskRun.ID,+ Updater: common.SystemBotEmail,+ Status: storepb.TaskRun_RUNNING,+ }); err != nil {+ return errors.Wrapf(err, "failed to update task run status to running")+ }+ s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{+ Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,+ TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{+ Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_WAITING,+ },+ })++ // Tickle the running scheduler to immediately pick up this newly RUNNING task+ // This prevents delays when user clicks "Run stage" in the UI+ select {+ case s.stateCfg.TaskRunTickleChan <- 0:+ default:+ // Channel is full, running scheduler will pick it up on next tick+ }++ return nil+}diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.gonew file mode 100644index 0000000000..572a1b0da3--- /dev/null+++ b/backend/runner/taskrun/running_scheduler.go@@ -0,0 +1,452 @@+package taskrun++import (+ "context"+ "fmt"+ "log/slog"+ "strconv"+ "sync"+ "time"++ "github.com/pkg/errors"+ "google.golang.org/protobuf/encoding/protojson"+ "google.golang.org/protobuf/types/known/timestamppb"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log"+ "github.com/bytebase/bytebase/backend/component/webhook"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+ "github.com/bytebase/bytebase/backend/store"+)++func getDatabaseKey(instanceID, databaseName string) string {+ return fmt.Sprintf("%s/%s", instanceID, databaseName)+}++// runRunningTaskRunsScheduler runs in a separate goroutine to schedule running task runs.+func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.WaitGroup) {+ ticker := time.NewTicker(taskSchedulerInterval)+ defer ticker.Stop()+ defer wg.Done()+ slog.Debug(fmt.Sprintf("Running task runs scheduler started and will run every %v", taskSchedulerInterval))+ for {+ select {+ case <-ticker.C:+ if err := s.scheduleRunningTaskRuns(ctx); err != nil {+ slog.Error("failed to schedule running task runs", log.BBError(err))+ }+ case <-s.stateCfg.TaskRunTickleChan:+ if err := s.scheduleRunningTaskRuns(ctx); err != nil {+ slog.Error("failed to schedule running task runs", log.BBError(err))+ }+ case <-ctx.Done():+ return+ }+ }+}++func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {+ taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{+ Status: &[]storepb.TaskRun_Status{storepb.TaskRun_RUNNING},+ })+ if err != nil {+ return errors.Wrapf(err, "failed to list pending tasks")+ }++ // Find the minimum task ID for each database.+ // We only run the first (i.e. which has the minimum task ID) task for each database.+ // 1. For ddl tasks, we run them one by one to get a sane schema dump and thus diff.+ // 2. For versioned tasks, this is our last resort to determine the order for tasks with the same version. We don't want to run them in parallel.+ // 2.1. Rollout 1 tasks will be run before rollout 2 tasks. Where, rollout 1 tasks are created before rollout 2 tasks.+ minTaskIDForDatabase := map[string]int{}+ for _, taskRun := range taskRuns {+ task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)+ if err != nil {+ slog.Error("failed to get task", slog.Int("task id", taskRun.TaskUID), log.BBError(err))+ continue+ }+ if task.DatabaseName == nil {+ continue+ }++ databaseKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)+ if isSequentialTask(task) {+ if _, ok := minTaskIDForDatabase[databaseKey]; !ok {+ minTaskIDForDatabase[databaseKey] = task.ID+ } else if minTaskIDForDatabase[databaseKey] > task.ID {+ minTaskIDForDatabase[databaseKey] = task.ID+ }+ }+ }++ for _, taskRun := range taskRuns {+ if err := s.scheduleRunningTaskRun(ctx, taskRun, minTaskIDForDatabase); err != nil {+ slog.Error("failed to schedule running task run", log.BBError(err))+ }+ }++ return nil+}++func (s *Scheduler) scheduleRunningTaskRun(ctx context.Context, taskRun *store.TaskRunMessage, minTaskIDForDatabase map[string]int) error {+ // Skip the task run if it is already executing.+ if _, ok := s.stateCfg.RunningTaskRuns.Load(taskRun.ID); ok {+ return nil+ }+ task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)+ if err != nil {+ return errors.Wrapf(err, "failed to get task")+ }+ if task.DatabaseName != nil && isSequentialTask(task) {+ // Skip the task run if there is an ongoing migration on the database.+ if taskUIDAny, ok := s.stateCfg.RunningDatabaseMigration.Load(getDatabaseKey(task.InstanceID, *task.DatabaseName)); ok {+ if taskUID, ok := taskUIDAny.(int); ok {+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{+ ReportTime: timestamppb.Now(),+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{+ Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{+ TaskUid: int32(taskUID),+ },+ },+ })+ }+ return nil+ }+ if taskUID := minTaskIDForDatabase[getDatabaseKey(task.InstanceID, *task.DatabaseName)]; taskUID != task.ID {+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{+ ReportTime: timestamppb.Now(),+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{+ Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{+ TaskUid: int32(taskUID),+ },+ },+ })+ return nil+ }+ }++ instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})+ if err != nil {+ return errors.Wrapf(err, "failed to get instance")+ }+ if instance.Deleted {+ return errors.Errorf("instance %v is deleted", task.InstanceID)+ }+ executor, ok := s.executorMap[task.Type]+ if !ok {+ return errors.Errorf("executor not found for task type: %v", task.Type)+ }++ // Check max connections per instance.+ maximumConnections := int(instance.Metadata.GetMaximumConnections())+ if maximumConnections <= 0 {+ maximumConnections = common.DefaultInstanceMaximumConnections+ }+ if s.stateCfg.InstanceOutstandingConnections.Increment(task.InstanceID, maximumConnections) {+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{+ ReportTime: timestamppb.Now(),+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{+ Cause: &storepb.SchedulerInfo_WaitingCause_ConnectionLimit{+ ConnectionLimit: true,+ },+ },+ })+ return nil+ }+ // decrement the connection count if we return below.+ revertInstanceConnectionsIncrement := true+ defer func() {+ if revertInstanceConnectionsIncrement {+ s.stateCfg.InstanceOutstandingConnections.Decrement(task.InstanceID)+ }+ }()++ // Check max running task runs per rollout.+ pipeline, err := s.store.GetPipelineByID(ctx, task.PipelineID)+ if err != nil {+ return errors.Wrapf(err, "failed to get pipeline")+ }+ if pipeline == nil {+ return errors.Errorf("pipeline %v not found", task.PipelineID)+ }++ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &pipeline.ProjectID})+ if err != nil {+ return errors.Wrapf(err, "failed to get project")+ }+ if project == nil {+ return errors.Errorf("project %v not found", pipeline.ProjectID)+ }++ rolloutID := strconv.Itoa(pipeline.ID)+ maxRunningTaskRunsPerRollout := int(project.Setting.GetParallelTasksPerRollout())+ if maxRunningTaskRunsPerRollout <= 0 {+ maxRunningTaskRunsPerRollout = defaultRolloutMaxRunningTaskRuns+ }+ if s.stateCfg.RolloutOutstandingTasks.Increment(rolloutID+"/"+task.InstanceID, maxRunningTaskRunsPerRollout) {+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{+ ReportTime: timestamppb.Now(),+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{+ Cause: &storepb.SchedulerInfo_WaitingCause_ParallelTasksLimit{+ ParallelTasksLimit: true,+ },+ },+ })+ return nil+ }++ // decrement the connection count if we return below.+ revertRolloutConnectionsIncrement := true+ defer func() {+ if revertRolloutConnectionsIncrement {+ s.stateCfg.RolloutOutstandingTasks.Decrement(rolloutID + "/" + task.InstanceID)+ }+ }()++ // Set taskrun StartAt when it's about to run.+ // So that the waiting time is not taken into account of the actual execution time.+ if err := s.store.UpdateTaskRunStartAt(ctx, taskRun.ID); err != nil {+ return errors.Wrapf(err, "failed to update task run start at")+ }++ // We MUST NOT return early below this line.+ // If we do want to return early, we must revert related states.+ s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)+ s.stateCfg.RunningTaskRuns.Store(taskRun.ID, true)+ if task.DatabaseName != nil {+ s.stateCfg.RunningDatabaseMigration.Store(getDatabaseKey(task.InstanceID, *task.DatabaseName), task.ID)+ }++ s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{+ Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,+ TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{+ Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_RUNNING,+ },+ })++ // We are sure that we will run the task.+ // The executor will decrement them.+ revertInstanceConnectionsIncrement = false+ revertRolloutConnectionsIncrement = false+ go s.runTaskRunOnce(ctx, taskRun, task, executor)+ return nil+}++func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMessage, task *store.TaskMessage, executor Executor) {+ defer func() {+ if r := recover(); r != nil {+ err, ok := r.(error)+ if !ok {+ err = errors.Errorf("%v", r)+ }+ slog.Error("Task scheduler V2 runTaskRunOnce PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))+ }+ }()+ defer func() {+ // We don't need to do s.stateCfg.RunningTaskRuns.Delete(taskRun.ID) to avoid race condition.+ s.stateCfg.RunningTaskRunsCancelFunc.Delete(taskRun.ID)+ if task.DatabaseName != nil {+ s.stateCfg.RunningDatabaseMigration.Delete(getDatabaseKey(task.InstanceID, *task.DatabaseName))+ }+ s.stateCfg.InstanceOutstandingConnections.Decrement(task.InstanceID)+ s.stateCfg.RolloutOutstandingTasks.Decrement(strconv.Itoa(task.PipelineID) + "/" + task.InstanceID)+ }()++ driverCtx, cancel := context.WithCancel(ctx)+ defer cancel()+ s.stateCfg.RunningTaskRunsCancelFunc.Store(taskRun.ID, cancel)++ done, result, err := RunExecutorOnce(ctx, driverCtx, executor, task, taskRun.ID)++ switch {+ case !done && err != nil:+ slog.Debug("Encountered transient error running task, will retry",+ slog.Int("id", task.ID),+ slog.String("type", task.Type.String()),+ log.BBError(err),+ )+ return++ case done && err != nil && errors.Is(err, context.Canceled):+ slog.Warn("task run is canceled",+ slog.Int("id", task.ID),+ slog.String("type", task.Type.String()),+ log.BBError(err),+ )+ resultBytes, marshalErr := protojson.Marshal(&storepb.TaskRunResult{+ Detail: "The task run is canceled",+ Changelog: "",+ Version: "",+ })+ if marshalErr != nil {+ slog.Error("Failed to marshal task run result",+ slog.Int("task_id", task.ID),+ slog.String("type", task.Type.String()),+ log.BBError(marshalErr),+ )+ return+ }+ code := common.Ok+ result := string(resultBytes)+ taskRunStatusPatch := &store.TaskRunStatusPatch{+ ID: taskRun.ID,+ Updater: common.SystemBotEmail,+ Status: storepb.TaskRun_CANCELED,+ Code: &code,+ Result: &result,+ }+ if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {+ slog.Error("Failed to mark task as CANCELED",+ slog.Int("id", task.ID),+ log.BBError(err),+ )+ return+ }+ return++ case done && err != nil:+ slog.Warn("task run failed",+ slog.Int("id", task.ID),+ slog.String("type", task.Type.String()),+ log.BBError(err),+ )+ taskRunResult := &storepb.TaskRunResult{+ Detail: err.Error(),+ Changelog: "",+ Version: "",+ }+ var errWithPosition *db.ErrorWithPosition+ if errors.As(err, &errWithPosition) {+ taskRunResult.StartPosition = errWithPosition.Start+ taskRunResult.EndPosition = errWithPosition.End+ }+ resultBytes, marshalErr := protojson.Marshal(taskRunResult)+ if marshalErr != nil {+ slog.Error("Failed to marshal task run result",+ slog.Int("task_id", task.ID),+ slog.String("type", task.Type.String()),+ log.BBError(marshalErr),+ )+ return+ }+ code := common.ErrorCode(err)+ result := string(resultBytes)+ taskRunStatusPatch := &store.TaskRunStatusPatch{+ ID: taskRun.ID,+ Updater: common.SystemBotEmail,+ Status: storepb.TaskRun_FAILED,+ Code: &code,+ Result: &result,+ }+ if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {+ slog.Error("Failed to mark task as FAILED",+ slog.Int("id", task.ID),+ log.BBError(err),+ )+ return+ }+ s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_FAILED, taskRunResult.Detail)+ return++ case done && err == nil:+ resultBytes, marshalErr := protojson.Marshal(result)+ if marshalErr != nil {+ slog.Error("Failed to marshal task run result",+ slog.Int("task_id", task.ID),+ slog.String("type", task.Type.String()),+ log.BBError(marshalErr),+ )+ return+ }+ code := common.Ok+ result := string(resultBytes)+ taskRunStatusPatch := &store.TaskRunStatusPatch{+ ID: taskRun.ID,+ Updater: common.SystemBotEmail,+ Status: storepb.TaskRun_DONE,+ Code: &code,+ Result: &result,+ }+ if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {+ slog.Error("Failed to mark task as DONE",+ slog.Int("id", task.ID),+ log.BBError(err),+ )+ return+ }+ s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_DONE, "")+ s.stateCfg.TaskSkippedOrDoneChan <- task.ID+ return+ default:+ // This case should not happen in normal flow, but adding for completeness+ slog.Error("Unexpected task execution state",+ slog.Int("id", task.ID),+ slog.String("type", task.Type.String()),+ slog.Bool("done", done),+ slog.Bool("has_error", err != nil),+ )+ return+ }+}++func (s *Scheduler) createActivityForTaskRunStatusUpdate(ctx context.Context, task *store.TaskMessage, newStatus storepb.TaskRun_Status, errDetail string) {+ if err := func() error {+ rollout, err := s.store.GetPipelineByID(ctx, task.PipelineID)+ if err != nil {+ return errors.Wrapf(err, "failed to get pipeline")+ }+ if rollout == nil {+ return errors.Errorf("pipeline %v not found", task.PipelineID)+ }+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &rollout.ProjectID})+ if err != nil {+ return errors.Wrapf(err, "failed to get project")+ }+ if project == nil {+ return errors.Errorf("project %v not found", rollout.ProjectID)+ }+ issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{+ PipelineID: &task.PipelineID,+ })+ if err != nil {+ return errors.Wrap(err, "failed to get issue")+ }+ s.webhookManager.CreateEvent(ctx, &webhook.Event{+ Actor: store.SystemBotUser,+ Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,+ Comment: "",+ Issue: webhook.NewIssue(issue),+ Rollout: webhook.NewRollout(rollout),+ Project: webhook.NewProject(project),+ TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{+ Title: task.GetDatabaseName(),+ Status: newStatus.String(),+ Detail: errDetail,+ },+ })+ return nil+ }(); err != nil {+ slog.Error("failed to create activity for task run status update", log.BBError(err))+ }+}++// isSequentialTask returns whether the task should be executed sequentially.+func isSequentialTask(task *store.TaskMessage) bool {+ //exhaustive:enforce+ switch task.Type {+ case storepb.Task_DATABASE_MIGRATE:+ // All DATABASE_MIGRATE operations (DDL/GHOST) should be sequential+ return true+ case storepb.Task_DATABASE_SDL:+ // SDL operations should be sequential+ return true+ case storepb.Task_DATABASE_CREATE,+ storepb.Task_DATABASE_EXPORT:+ return false+ case storepb.Task_TASK_TYPE_UNSPECIFIED:+ return false+ default:+ return false+ }+}diff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.goindex ed5ebb8fb2..af0c5d78e6 100644--- a/backend/runner/taskrun/scheduler.go+++ b/backend/runner/taskrun/scheduler.go@@ -2,15 +2,11 @@ package taskrunimport ("context"- "fmt""log/slog"- "strconv""sync""time""github.com/pkg/errors"- "google.golang.org/protobuf/encoding/protojson"- "google.golang.org/protobuf/types/known/timestamppb""github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log"@@ -19,9 +15,7 @@ import ("github.com/bytebase/bytebase/backend/component/webhook""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/store"- "github.com/bytebase/bytebase/backend/utils")const (@@ -73,621 +67,18 @@ func (s *Scheduler) Register(taskType storepb.Task_Type, executorGetter Executor// Run will start the scheduler.func (s *Scheduler) Run(ctx context.Context, wg *sync.WaitGroup) {- go s.ListenTaskSkippedOrDone(ctx)-- ticker := time.NewTicker(taskSchedulerInterval)- defer ticker.Stop()defer wg.Done()- slog.Debug(fmt.Sprintf("Task scheduler V2 started and will run every %v", taskSchedulerInterval))- for {- select {- case <-ticker.C:- s.runOnce(ctx)- case <-s.stateCfg.TaskRunTickleChan:- s.runOnce(ctx)- case <-ctx.Done():- return- }- }-}--func (s *Scheduler) runOnce(ctx context.Context) {- defer func() {- if r := recover(); r != nil {- err, ok := r.(error)- if !ok {- err = errors.Errorf("%v", r)- }- slog.Error("Task scheduler V2 PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))- }- }()-- if err := s.scheduleAutoRolloutTasks(ctx); err != nil {- slog.Error("failed to schedule auto rollout tasks", log.BBError(err))- }-- if err := s.schedulePendingTaskRuns(ctx); err != nil {- slog.Error("failed to schedule pending task runs", log.BBError(err))- }-- if err := s.scheduleRunningTaskRuns(ctx); err != nil {- slog.Error("failed to schedule running task runs", log.BBError(err))- }-}--func (s *Scheduler) scheduleAutoRolloutTasks(ctx context.Context) error {- environments, err := s.store.GetEnvironment(ctx)- if err != nil {- return errors.Wrapf(err, "failed to list environments")- }-- var envs []string- for _, environment := range environments.GetEnvironments() {- policy, err := s.store.GetRolloutPolicy(ctx, environment.Id)- if err != nil {- return errors.Wrapf(err, "failed to get rollout policy for environment %s", environment.Id)- }- if policy.Automatic {- envs = append(envs, environment.Id)- }- }- taskIDs, err := s.store.ListTasksToAutoRollout(ctx, envs)- if err != nil {- return errors.Wrapf(err, "failed to list tasks with zero task run")- }- for _, taskID := range taskIDs {- if err := s.scheduleAutoRolloutTask(ctx, taskID); err != nil {- slog.Error("failed to schedule auto rollout task", log.BBError(err))- }- }-- return nil-}--func (s *Scheduler) scheduleAutoRolloutTask(ctx context.Context, taskUID int) error {- task, err := s.store.GetTaskByID(ctx, taskUID)- if err != nil {- return errors.Wrapf(err, "failed to get task")- }- if task == nil {- return nil- }-- pipeline, err := s.store.GetPipelineByID(ctx, task.PipelineID)- if err != nil {- return errors.Wrapf(err, "failed to get pipeline")- }- if pipeline == nil {- return errors.Errorf("pipeline %v not found", task.PipelineID)- }-- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &pipeline.ProjectID})- if err != nil {- return errors.Wrapf(err, "failed to get project")- }- if project == nil {- return errors.Errorf("project %v not found", pipeline.ProjectID)- }-- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})- if err != nil {- return err- }- if instance.Deleted {- return nil- }-- issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PipelineID: &task.PipelineID})- if err != nil {- return err- }- if issue != nil {- if issue.Status != storepb.Issue_OPEN {- return nil- }- // Check if issue approval is required according to the project settings- if project.Setting.RequireIssueApproval {- approved, err := utils.CheckIssueApproved(issue)- if err != nil {- return errors.Wrapf(err, "failed to check if the issue is approved")- }- if !approved {- return nil- }- }- }-- // Check the latest plan checks based on project settings (error only)- if project.Setting.RequirePlanCheckNoError {- pass, err := func() (bool, error) {- plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{PipelineID: &task.PipelineID})- if err != nil {- return false, errors.Wrapf(err, "failed to get plan")- }- if plan == nil {- return true, nil- }- latestRuns, err := s.store.ListPlanCheckRuns(ctx, &store.FindPlanCheckRunMessage{- PlanUID: &plan.UID,- })- if err != nil {- return false, errors.Wrapf(err, "failed to list latest plan check runs")- }- for _, run := range latestRuns {- if run.Status != store.PlanCheckRunStatusDone {- return false, nil- }- for _, result := range run.Result.Results {- if result.Status == storepb.Advice_ERROR {- return false, nil- }- }- }- return true, nil- }()- if err != nil {- return errors.Wrapf(err, "failed to check if plan check passes")- }- if !pass {- return nil- }- }-- create := &store.TaskRunMessage{- TaskUID: task.ID,- }- if task.Payload.GetSheetId() != 0 {- sheetUID := int(task.Payload.GetSheetId())- create.SheetUID = &sheetUID- }-- if err := s.store.CreatePendingTaskRuns(ctx, common.SystemBotEmail, create); err != nil {- return errors.Wrapf(err, "failed to create pending task runs")- }-- s.webhookManager.CreateEvent(ctx, &webhook.Event{- Actor: store.SystemBotUser,- Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,- Comment: "",- Issue: webhook.NewIssue(issue),- Rollout: webhook.NewRollout(pipeline),- Project: webhook.NewProject(project),- TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{- Title: task.GetDatabaseName(),- Status: storepb.TaskRun_PENDING.String(),- },- })-- return nil-}--func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {- taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_PENDING},- })- if err != nil {- return errors.Wrapf(err, "failed to list pending tasks")- }- for _, taskRun := range taskRuns {- if err := s.schedulePendingTaskRun(ctx, taskRun); err != nil {- slog.Error("failed to schedule pending task run", log.BBError(err))- }- }-- return nil-}--func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {- // here, we move pending taskruns to running taskruns which means they are ready to be executed.- // pending taskruns remain pending if- // 1. taskRun.RunAt not met.- // 2. for versioned tasks, there are other versioned tasks on the same database with- // a smaller version not finished yet. we need to wait for those first.- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)- if err != nil {- return errors.Wrapf(err, "failed to get task")- }- if taskRun.RunAt != nil && time.Now().Before(*taskRun.RunAt) {- return nil- }-- doSchedule, err := func() (bool, error) {- if task.DatabaseName == nil {- return true, nil- }-- schemaVersion := task.Payload.GetSchemaVersion()- if schemaVersion == "" {- return true, nil- }-- maybeTaskID, err := s.store.FindBlockingTaskByVersion(ctx, task.PipelineID, task.InstanceID, *task.DatabaseName, schemaVersion)- if err != nil {- return false, errors.Wrapf(err, "failed to find blocking versioned tasks")- }- if maybeTaskID != nil {- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{- ReportTime: timestamppb.Now(),- WaitingCause: &storepb.SchedulerInfo_WaitingCause{- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{- TaskUid: int32(*maybeTaskID),- },- },- })- return false, nil- }- s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)- return true, nil- }()- if err != nil {- return errors.Wrapf(err, "failed to check blocking versioned tasks")- }- if !doSchedule {- return nil- }- if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{- ID: taskRun.ID,- Updater: common.SystemBotEmail,- Status: storepb.TaskRun_RUNNING,- }); err != nil {- return errors.Wrapf(err, "failed to update task run status to running")- }- s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{- Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,- TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{- Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_WAITING,- },- })- return nil-}--func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {- taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_RUNNING},- })- if err != nil {- return errors.Wrapf(err, "failed to list pending tasks")- }-- // Find the minimum task ID for each database.- // We only run the first (i.e. which has the minimum task ID) task for each database.- // 1. For ddl tasks, we run them one by one to get a sane schema dump and thus diff.- // 2. For versioned tasks, this is our last resort to determine the order for tasks with the same version. We don't want to run them in parallel.- // 2.1. Rollout 1 tasks will be run before rollout 2 tasks. Where, rollout 1 tasks are created before rollout 2 tasks.- minTaskIDForDatabase := map[string]int{}- for _, taskRun := range taskRuns {- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)- if err != nil {- slog.Error("failed to get task", slog.Int("task id", taskRun.TaskUID), log.BBError(err))- continue- }- if task.DatabaseName == nil {- continue- }-- databaseKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)- if isSequentialTask(task) {- if _, ok := minTaskIDForDatabase[databaseKey]; !ok {- minTaskIDForDatabase[databaseKey] = task.ID- } else if minTaskIDForDatabase[databaseKey] > task.ID {- minTaskIDForDatabase[databaseKey] = task.ID- }- }- }-- for _, taskRun := range taskRuns {- if err := s.scheduleRunningTaskRun(ctx, taskRun, minTaskIDForDatabase); err != nil {- slog.Error("failed to schedule running task run", log.BBError(err))- }- }-- return nil-}--func (s *Scheduler) scheduleRunningTaskRun(ctx context.Context, taskRun *store.TaskRunMessage, minTaskIDForDatabase map[string]int) error {- // Skip the task run if it is already executing.- if _, ok := s.stateCfg.RunningTaskRuns.Load(taskRun.ID); ok {- return nil- }- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)- if err != nil {- return errors.Wrapf(err, "failed to get task")- }- if task.DatabaseName != nil && isSequentialTask(task) {- // Skip the task run if there is an ongoing migration on the database.- if taskUIDAny, ok := s.stateCfg.RunningDatabaseMigration.Load(getDatabaseKey(task.InstanceID, *task.DatabaseName)); ok {- if taskUID, ok := taskUIDAny.(int); ok {- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{- ReportTime: timestamppb.Now(),- WaitingCause: &storepb.SchedulerInfo_WaitingCause{- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{- TaskUid: int32(taskUID),- },- },- })- }- return nil- }- if taskUID := minTaskIDForDatabase[getDatabaseKey(task.InstanceID, *task.DatabaseName)]; taskUID != task.ID {- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{- ReportTime: timestamppb.Now(),- WaitingCause: &storepb.SchedulerInfo_WaitingCause{- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{- TaskUid: int32(taskUID),- },- },- })- return nil- }- }-- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})- if err != nil {- return errors.Wrapf(err, "failed to get instance")- }- if instance.Deleted {- return errors.Errorf("instance %v is deleted", task.InstanceID)- }- executor, ok := s.executorMap[task.Type]- if !ok {- return errors.Errorf("executor not found for task type: %v", task.Type)- }-- // Check max connections per instance.- maximumConnections := int(instance.Metadata.GetMaximumConnections())- if maximumConnections <= 0 {- maximumConnections = common.DefaultInstanceMaximumConnections- }- if s.stateCfg.InstanceOutstandingConnections.Increment(task.InstanceID, maximumConnections) {- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{- ReportTime: timestamppb.Now(),- WaitingCause: &storepb.SchedulerInfo_WaitingCause{- Cause: &storepb.SchedulerInfo_WaitingCause_ConnectionLimit{- ConnectionLimit: true,- },- },- })- return nil- }- // decrement the connection count if we return below.- revertInstanceConnectionsIncrement := true- defer func() {- if revertInstanceConnectionsIncrement {- s.stateCfg.InstanceOutstandingConnections.Decrement(task.InstanceID)- }- }()-- // Check max running task runs per rollout.- pipeline, err := s.store.GetPipelineByID(ctx, task.PipelineID)- if err != nil {- return errors.Wrapf(err, "failed to get pipeline")- }- if pipeline == nil {- return errors.Errorf("pipeline %v not found", task.PipelineID)- }-- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &pipeline.ProjectID})- if err != nil {- return errors.Wrapf(err, "failed to get project")- }- if project == nil {- return errors.Errorf("project %v not found", pipeline.ProjectID)- }-- rolloutID := strconv.Itoa(pipeline.ID)- maxRunningTaskRunsPerRollout := int(project.Setting.GetParallelTasksPerRollout())- if maxRunningTaskRunsPerRollout <= 0 {- maxRunningTaskRunsPerRollout = defaultRolloutMaxRunningTaskRuns- }- if s.stateCfg.RolloutOutstandingTasks.Increment(rolloutID+"/"+task.InstanceID, maxRunningTaskRunsPerRollout) {- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{- ReportTime: timestamppb.Now(),- WaitingCause: &storepb.SchedulerInfo_WaitingCause{- Cause: &storepb.SchedulerInfo_WaitingCause_ParallelTasksLimit{- ParallelTasksLimit: true,- },- },- })- return nil- }-- // decrement the connection count if we return below.- revertRolloutConnectionsIncrement := true- defer func() {- if revertRolloutConnectionsIncrement {- s.stateCfg.RolloutOutstandingTasks.Decrement(rolloutID + "/" + task.InstanceID)- }- }()-- // Set taskrun StartAt when it's about to run.- // So that the waiting time is not taken into account of the actual execution time.- if err := s.store.UpdateTaskRunStartAt(ctx, taskRun.ID); err != nil {- return errors.Wrapf(err, "failed to update task run start at")- }-- // We MUST NOT return early below this line.- // If we do want to return early, we must revert related states.- s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)- s.stateCfg.RunningTaskRuns.Store(taskRun.ID, true)- if task.DatabaseName != nil {- s.stateCfg.RunningDatabaseMigration.Store(getDatabaseKey(task.InstanceID, *task.DatabaseName), task.ID)- }-- s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{- Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,- TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{- Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_RUNNING,- },- })-- // We are sure that we will run the task.- // The executor will decrement them.- revertInstanceConnectionsIncrement = false- revertRolloutConnectionsIncrement = false- go s.runTaskRunOnce(ctx, taskRun, task, executor)- return nil-}--func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMessage, task *store.TaskMessage, executor Executor) {- defer func() {- if r := recover(); r != nil {- err, ok := r.(error)- if !ok {- err = errors.Errorf("%v", r)- }- slog.Error("Task scheduler V2 runTaskRunOnce PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))- }- }()- defer func() {- // We don't need to do s.stateCfg.RunningTaskRuns.Delete(taskRun.ID) to avoid race condition.- s.stateCfg.RunningTaskRunsCancelFunc.Delete(taskRun.ID)- if task.DatabaseName != nil {- s.stateCfg.RunningDatabaseMigration.Delete(getDatabaseKey(task.InstanceID, *task.DatabaseName))- }- s.stateCfg.InstanceOutstandingConnections.Decrement(task.InstanceID)- s.stateCfg.RolloutOutstandingTasks.Decrement(strconv.Itoa(task.PipelineID) + "/" + task.InstanceID)- }()+ go s.ListenTaskSkippedOrDone(ctx)- driverCtx, cancel := context.WithCancel(ctx)- defer cancel()- s.stateCfg.RunningTaskRunsCancelFunc.Store(taskRun.ID, cancel)-- done, result, err := RunExecutorOnce(ctx, driverCtx, executor, task, taskRun.ID)-- switch {- case !done && err != nil:- slog.Debug("Encountered transient error running task, will retry",- slog.Int("id", task.ID),- slog.String("type", task.Type.String()),- log.BBError(err),- )- return-- case done && err != nil && errors.Is(err, context.Canceled):- slog.Warn("task run is canceled",- slog.Int("id", task.ID),- slog.String("type", task.Type.String()),- log.BBError(err),- )- resultBytes, marshalErr := protojson.Marshal(&storepb.TaskRunResult{- Detail: "The task run is canceled",- Changelog: "",- Version: "",- })- if marshalErr != nil {- slog.Error("Failed to marshal task run result",- slog.Int("task_id", task.ID),- slog.String("type", task.Type.String()),- log.BBError(marshalErr),- )- return- }- code := common.Ok- result := string(resultBytes)- taskRunStatusPatch := &store.TaskRunStatusPatch{- ID: taskRun.ID,- Updater: common.SystemBotEmail,- Status: storepb.TaskRun_CANCELED,- Code: &code,- Result: &result,- }- if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {- slog.Error("Failed to mark task as CANCELED",- slog.Int("id", task.ID),- log.BBError(err),- )- return- }- return-- case done && err != nil:- slog.Warn("task run failed",- slog.Int("id", task.ID),- slog.String("type", task.Type.String()),- log.BBError(err),- )- taskRunResult := &storepb.TaskRunResult{- Detail: err.Error(),- Changelog: "",- Version: "",- }- var errWithPosition *db.ErrorWithPosition- if errors.As(err, &errWithPosition) {- taskRunResult.StartPosition = errWithPosition.Start- taskRunResult.EndPosition = errWithPosition.End- }- resultBytes, marshalErr := protojson.Marshal(taskRunResult)- if marshalErr != nil {- slog.Error("Failed to marshal task run result",- slog.Int("task_id", task.ID),- slog.String("type", task.Type.String()),- log.BBError(marshalErr),- )- return- }- code := common.ErrorCode(err)- result := string(resultBytes)- taskRunStatusPatch := &store.TaskRunStatusPatch{- ID: taskRun.ID,- Updater: common.SystemBotEmail,- Status: storepb.TaskRun_FAILED,- Code: &code,- Result: &result,- }- if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {- slog.Error("Failed to mark task as FAILED",- slog.Int("id", task.ID),- log.BBError(err),- )- return- }- s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_FAILED, taskRunResult.Detail)- return-- case done && err == nil:- resultBytes, marshalErr := protojson.Marshal(result)- if marshalErr != nil {- slog.Error("Failed to marshal task run result",- slog.Int("task_id", task.ID),- slog.String("type", task.Type.String()),- log.BBError(marshalErr),- )- return- }- code := common.Ok- result := string(resultBytes)- taskRunStatusPatch := &store.TaskRunStatusPatch{- ID: taskRun.ID,- Updater: common.SystemBotEmail,- Status: storepb.TaskRun_DONE,- Code: &code,- Result: &result,- }- if _, err := s.store.UpdateTaskRunStatus(ctx, taskRunStatusPatch); err != nil {- slog.Error("Failed to mark task as DONE",- slog.Int("id", task.ID),- log.BBError(err),- )- return- }- s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_DONE, "")- s.stateCfg.TaskSkippedOrDoneChan <- task.ID- return- default:- // This case should not happen in normal flow, but adding for completeness- slog.Error("Unexpected task execution state",- slog.Int("id", task.ID),- slog.String("type", task.Type.String()),- slog.Bool("done", done),- slog.Bool("has_error", err != nil),- )- return- }-}+ // Start three independent schedulers+ wg.Add(3)+ go s.runAutoRolloutScheduler(ctx, wg)+ go s.runPendingTaskRunsScheduler(ctx, wg)+ go s.runRunningTaskRunsScheduler(ctx, wg)-func getDatabaseKey(instanceID, databaseName string) string {- return fmt.Sprintf("%s/%s", instanceID, databaseName)+ slog.Debug("Task scheduler V2 started with independent runners")+ <-ctx.Done()}func (s *Scheduler) ListenTaskSkippedOrDone(ctx context.Context) {@@ -893,47 +284,6 @@ func (s *Scheduler) ListenTaskSkippedOrDone(ctx context.Context) {}}-func (s *Scheduler) createActivityForTaskRunStatusUpdate(ctx context.Context, task *store.TaskMessage, newStatus storepb.TaskRun_Status, errDetail string) {- if err := func() error {- rollout, err := s.store.GetPipelineByID(ctx, task.PipelineID)- if err != nil {- return errors.Wrapf(err, "failed to get pipeline")- }- if rollout == nil {- return errors.Errorf("pipeline %v not found", task.PipelineID)- }- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &rollout.ProjectID})- if err != nil {- return errors.Wrapf(err, "failed to get project")- }- if project == nil {- return errors.Errorf("project %v not found", rollout.ProjectID)- }- issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{- PipelineID: &task.PipelineID,- })- if err != nil {- return errors.Wrap(err, "failed to get issue")- }- s.webhookManager.CreateEvent(ctx, &webhook.Event{- Actor: store.SystemBotUser,- Type: storepb.Activity_ISSUE_PIPELINE_TASK_RUN_STATUS_UPDATE,- Comment: "",- Issue: webhook.NewIssue(issue),- Rollout: webhook.NewRollout(rollout),- Project: webhook.NewProject(project),- TaskRunStatusUpdate: &webhook.EventTaskRunStatusUpdate{- Title: task.GetDatabaseName(),- Status: newStatus.String(),- Detail: errDetail,- },- })- return nil- }(); err != nil {- slog.Error("failed to create activity for task run status update", log.BBError(err))- }-}-func tasksSkippedOrDone(tasks []*store.TaskMessage) bool {for _, task := range tasks {skipped := task.Payload.GetSkipped()@@ -945,26 +295,6 @@ func tasksSkippedOrDone(tasks []*store.TaskMessage) bool {return true}-// isSequentialTask returns whether the task should be executed sequentially.-func isSequentialTask(task *store.TaskMessage) bool {- //exhaustive:enforce- switch task.Type {- case storepb.Task_DATABASE_MIGRATE:- // All DATABASE_MIGRATE operations (DDL/GHOST) should be sequential- return true- case storepb.Task_DATABASE_SDL:- // SDL operations should be sequential- return true- case storepb.Task_DATABASE_CREATE,- storepb.Task_DATABASE_EXPORT:- return false- case storepb.Task_TASK_TYPE_UNSPECIFIED:- return false- default:- return false- }-}-// getAllEnvironmentIDs returns all environment IDs from the store.func getAllEnvironmentIDs(ctx context.Context, s *store.Store) ([]string, error) {environments, err := s.GetEnvironment(ctx)