Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(taskrun): give each task run scheduler its own tickle channel (#21355)
Danny Xu5ea5b192d59c
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.goindex 5c8c5f9e86..9b9f2d1e4c 100644--- a/backend/api/v1/rollout_service.go+++ b/backend/api/v1/rollout_service.go@@ -336,8 +336,8 @@ func (s *RolloutService) CreateRollout(ctx context.Context, req *connect.Requestreturn nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert to rollout"))}- // Tickle task run scheduler.- s.bus.TaskRunTickleChan <- 0+ // Tickle the pending task run scheduler.+ bus.Tickle(s.bus.TaskRunPendingTickleChan)return connect.NewResponse(rolloutV1), nil}@@ -930,8 +930,8 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Requestreturn nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create pending task runs, error %v", err))}- // Tickle task run scheduler.- s.bus.TaskRunTickleChan <- 0+ // Tickle the pending task run scheduler.+ bus.Tickle(s.bus.TaskRunPendingTickleChan)return connect.NewResponse(&v1pb.BatchRunTasksResponse{}), nil}diff --git a/backend/component/bus/bus.go b/backend/component/bus/bus.goindex bf8e7cf09a..729d08d346 100644--- a/backend/component/bus/bus.go+++ b/backend/component/bus/bus.go@@ -44,8 +44,18 @@ type Bus struct {// PlanCheckTickleChan is the tickler for plan check scheduler.PlanCheckTickleChan chan int- // TaskRunTickleChan is the tickler for task run scheduler.- TaskRunTickleChan chan int+ // TaskRunPendingTickleChan is the tickler for the pending task run+ // scheduler: a task run was created and needs scheduling.+ TaskRunPendingTickleChan chan int+ // TaskRunRunningTickleChan is the tickler for the running task run+ // scheduler: a task run became available and needs to be picked up.+ //+ // The two task run schedulers must not share one channel. A channel value is+ // delivered to exactly one receiver, so a shared channel hands each wake-up+ // to whichever scheduler happens to be ready for it rather than to the one+ // the sender meant, and the scheduler that actually had work waits out its+ // 5s ticker instead. That cost a measured ~5s per task run transition.+ TaskRunRunningTickleChan chan int// ReviewRunTickleChan is the tickler for the review run scheduler.ReviewRunTickleChan chan int@@ -58,11 +68,23 @@ type Bus struct {func New() (*Bus, error) {return &Bus{- ApprovalCheckChan: make(chan IssueRef, 1000),- PlanCheckTickleChan: make(chan int, 1000),- TaskRunTickleChan: make(chan int, 1000),- ReviewRunTickleChan: make(chan int, 1000),- RolloutCreationChan: make(chan PlanRef, 100),- PlanCompletionCheckChan: make(chan PlanRef, 1000),+ ApprovalCheckChan: make(chan IssueRef, 1000),+ PlanCheckTickleChan: make(chan int, 1000),+ TaskRunPendingTickleChan: make(chan int, 1000),+ TaskRunRunningTickleChan: make(chan int, 1000),+ ReviewRunTickleChan: make(chan int, 1000),+ RolloutCreationChan: make(chan PlanRef, 100),+ PlanCompletionCheckChan: make(chan PlanRef, 1000),}, nil}++// Tickle wakes a scheduler without blocking. A tickle carries no information+// beyond "there may be work"; every scheduler sweeps the store for all of it,+// so a wake-up already queued makes another redundant. Not blocking here also+// keeps a request handler from parking on a scheduler that has stopped reading.+func Tickle(ch chan int) {+ select {+ case ch <- 0:+ default:+ }+}diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.goindex 3b476a49a0..f507789a5e 100644--- a/backend/runner/taskrun/pending_scheduler.go+++ b/backend/runner/taskrun/pending_scheduler.go@@ -13,6 +13,7 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log"+ "github.com/bytebase/bytebase/backend/component/bus""github.com/bytebase/bytebase/backend/component/productmetrics"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store"@@ -40,7 +41,7 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Waif err := s.schedulePendingTaskRuns(ctx); err != nil {slog.Error("failed to schedule pending task runs", log.BBError(err))}- case <-s.bus.TaskRunTickleChan:+ case <-s.bus.TaskRunPendingTickleChan:if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {// Grace period is tracked by the ticker branch; just skip here.continue@@ -247,10 +248,9 @@ func (s *Scheduler) promoteTaskRun(ctx context.Context, taskRun *store.TaskRunMereturn errors.Wrapf(err, "failed to update task run status to available")}- select {- case s.bus.TaskRunTickleChan <- 0:- default:- }+ // The task run is available now, which is the running scheduler's business,+ // not this one's.+ bus.Tickle(s.bus.TaskRunRunningTickleChan)return nil}diff --git a/backend/runner/taskrun/rollout_creator.go b/backend/runner/taskrun/rollout_creator.goindex 0302d4749c..808af50c88 100644--- a/backend/runner/taskrun/rollout_creator.go+++ b/backend/runner/taskrun/rollout_creator.go@@ -205,8 +205,8 @@ func (rc *RolloutCreator) tryCreateRollout(ctx context.Context, ref bus.PlanRef)return}- // Tickle task run scheduler- rc.bus.TaskRunTickleChan <- 0+ // Tickle the pending task run scheduler.+ bus.Tickle(rc.bus.TaskRunPendingTickleChan)slog.Info("successfully auto-created rollout", slog.String("project", ref.ProjectID), slog.Int("plan_id", int(planID)))}diff --git a/backend/runner/taskrun/rollout_creator_test.go b/backend/runner/taskrun/rollout_creator_test.goindex 73608da8b9..fdbd99d56f 100644--- a/backend/runner/taskrun/rollout_creator_test.go+++ b/backend/runner/taskrun/rollout_creator_test.go@@ -120,7 +120,7 @@ func TestTryCreateRolloutSkipsDraft(t *testing.T) {require.NotNil(t, gotIssue)require.True(t, gotIssue.Payload.GetDraft())require.Equal(t, storepb.Issue_OPEN, gotIssue.Status)- require.Empty(t, b.TaskRunTickleChan)+ require.Empty(t, b.TaskRunPendingTickleChan)}func TestTryCreateRolloutSkipsArchivedProject(t *testing.T) {@@ -204,7 +204,7 @@ func TestTryCreateRolloutSkipsArchivedProject(t *testing.T) {tasks, err := s.ListTasks(ctx, &store.TaskFind{ProjectID: plan.ProjectID, PlanID: &plan.UID})require.NoError(t, err)require.Empty(t, tasks)- require.Empty(t, b.TaskRunTickleChan)+ require.Empty(t, b.TaskRunPendingTickleChan)}func TestTryCreateRolloutResolvesProjectInstanceOutsideRequestContext(t *testing.T) {@@ -277,7 +277,7 @@ func TestTryCreateRolloutResolvesProjectInstanceOutsideRequestContext(t *testingtasks, err := s.ListTasks(ctx, &store.TaskFind{ProjectID: projectID, PlanID: &plan.UID})require.NoError(t, err)require.Len(t, tasks, 1)- require.Len(t, b.TaskRunTickleChan, 1)+ require.Len(t, b.TaskRunPendingTickleChan, 1)}func TestTryCreateRolloutRejectsArchivedProjectInstanceWithWarmDatabaseCache(t *testing.T) {@@ -367,7 +367,7 @@ func TestTryCreateRolloutRejectsArchivedProjectInstanceWithWarmDatabaseCache(t *tasks, err := s.ListTasks(ctx, &store.TaskFind{ProjectID: projectID, PlanID: &plan.UID})require.NoError(t, err)require.Empty(t, tasks)- require.Empty(t, b.TaskRunTickleChan)+ require.Empty(t, b.TaskRunPendingTickleChan)}func setupRolloutCreatorStore(ctx context.Context, t *testing.T) *store.Store {diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.goindex d49b3b893f..15c7741411 100644--- a/backend/runner/taskrun/running_scheduler.go+++ b/backend/runner/taskrun/running_scheduler.go@@ -38,7 +38,7 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Waif err := s.scheduleRunningTaskRuns(ctx); err != nil {slog.Error("failed to schedule running task runs", log.BBError(err))}- case <-s.bus.TaskRunTickleChan:+ case <-s.bus.TaskRunRunningTickleChan:if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {continue}