Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(taskrun): give each task run scheduler its own tickle channel (#21355)

Danny Xu5ea5b192d59c
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.go
index 5c8c5f9e86..9b9f2d1e4c 100644
--- a/backend/api/v1/rollout_service.go
+++ b/backend/api/v1/rollout_service.go
@@ -336,8 +336,8 @@ func (s *RolloutService) CreateRollout(ctx context.Context, req *connect.Request
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert to rollout"))
}
- // Tickle task run scheduler.
- s.bus.TaskRunTickleChan <- 0
+ // Tickle the pending task run scheduler.
+ bus.Tickle(s.bus.TaskRunPendingTickleChan)
return connect.NewResponse(rolloutV1), nil
}
@@ -930,8 +930,8 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create pending task runs, error %v", err))
}
- // Tickle task run scheduler.
- s.bus.TaskRunTickleChan <- 0
+ // Tickle the pending task run scheduler.
+ bus.Tickle(s.bus.TaskRunPendingTickleChan)
return connect.NewResponse(&v1pb.BatchRunTasksResponse{}), nil
}
diff --git a/backend/component/bus/bus.go b/backend/component/bus/bus.go
index bf8e7cf09a..729d08d346 100644
--- a/backend/component/bus/bus.go
+++ b/backend/component/bus/bus.go
@@ -44,8 +44,18 @@ type Bus struct {
// PlanCheckTickleChan is the tickler for plan check scheduler.
PlanCheckTickleChan chan int
- // TaskRunTickleChan is the tickler for task run scheduler.
- TaskRunTickleChan chan int
+ // TaskRunPendingTickleChan is the tickler for the pending task run
+ // scheduler: a task run was created and needs scheduling.
+ TaskRunPendingTickleChan chan int
+ // TaskRunRunningTickleChan is the tickler for the running task run
+ // scheduler: a task run became available and needs to be picked up.
+ //
+ // The two task run schedulers must not share one channel. A channel value is
+ // delivered to exactly one receiver, so a shared channel hands each wake-up
+ // to whichever scheduler happens to be ready for it rather than to the one
+ // the sender meant, and the scheduler that actually had work waits out its
+ // 5s ticker instead. That cost a measured ~5s per task run transition.
+ TaskRunRunningTickleChan chan int
// ReviewRunTickleChan is the tickler for the review run scheduler.
ReviewRunTickleChan chan int
@@ -58,11 +68,23 @@ type Bus struct {
func New() (*Bus, error) {
return &Bus{
- ApprovalCheckChan: make(chan IssueRef, 1000),
- PlanCheckTickleChan: make(chan int, 1000),
- TaskRunTickleChan: make(chan int, 1000),
- ReviewRunTickleChan: make(chan int, 1000),
- RolloutCreationChan: make(chan PlanRef, 100),
- PlanCompletionCheckChan: make(chan PlanRef, 1000),
+ ApprovalCheckChan: make(chan IssueRef, 1000),
+ PlanCheckTickleChan: make(chan int, 1000),
+ TaskRunPendingTickleChan: make(chan int, 1000),
+ TaskRunRunningTickleChan: make(chan int, 1000),
+ ReviewRunTickleChan: make(chan int, 1000),
+ RolloutCreationChan: make(chan PlanRef, 100),
+ PlanCompletionCheckChan: make(chan PlanRef, 1000),
}, nil
}
+
+// Tickle wakes a scheduler without blocking. A tickle carries no information
+// beyond "there may be work"; every scheduler sweeps the store for all of it,
+// so a wake-up already queued makes another redundant. Not blocking here also
+// keeps a request handler from parking on a scheduler that has stopped reading.
+func Tickle(ch chan int) {
+ select {
+ case ch <- 0:
+ default:
+ }
+}
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
index 3b476a49a0..f507789a5e 100644
--- a/backend/runner/taskrun/pending_scheduler.go
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -13,6 +13,7 @@ import (
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/component/bus"
"github.com/bytebase/bytebase/backend/component/productmetrics"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
@@ -40,7 +41,7 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Wa
if err := s.schedulePendingTaskRuns(ctx); err != nil {
slog.Error("failed to schedule pending task runs", log.BBError(err))
}
- case <-s.bus.TaskRunTickleChan:
+ case <-s.bus.TaskRunPendingTickleChan:
if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {
// Grace period is tracked by the ticker branch; just skip here.
continue
@@ -247,10 +248,9 @@ func (s *Scheduler) promoteTaskRun(ctx context.Context, taskRun *store.TaskRunMe
return errors.Wrapf(err, "failed to update task run status to available")
}
- select {
- case s.bus.TaskRunTickleChan <- 0:
- default:
- }
+ // The task run is available now, which is the running scheduler's business,
+ // not this one's.
+ bus.Tickle(s.bus.TaskRunRunningTickleChan)
return nil
}
diff --git a/backend/runner/taskrun/rollout_creator.go b/backend/runner/taskrun/rollout_creator.go
index 0302d4749c..808af50c88 100644
--- a/backend/runner/taskrun/rollout_creator.go
+++ b/backend/runner/taskrun/rollout_creator.go
@@ -205,8 +205,8 @@ func (rc *RolloutCreator) tryCreateRollout(ctx context.Context, ref bus.PlanRef)
return
}
- // Tickle task run scheduler
- rc.bus.TaskRunTickleChan <- 0
+ // Tickle the pending task run scheduler.
+ bus.Tickle(rc.bus.TaskRunPendingTickleChan)
slog.Info("successfully auto-created rollout", slog.String("project", ref.ProjectID), slog.Int("plan_id", int(planID)))
}
diff --git a/backend/runner/taskrun/rollout_creator_test.go b/backend/runner/taskrun/rollout_creator_test.go
index 73608da8b9..fdbd99d56f 100644
--- a/backend/runner/taskrun/rollout_creator_test.go
+++ b/backend/runner/taskrun/rollout_creator_test.go
@@ -120,7 +120,7 @@ func TestTryCreateRolloutSkipsDraft(t *testing.T) {
require.NotNil(t, gotIssue)
require.True(t, gotIssue.Payload.GetDraft())
require.Equal(t, storepb.Issue_OPEN, gotIssue.Status)
- require.Empty(t, b.TaskRunTickleChan)
+ require.Empty(t, b.TaskRunPendingTickleChan)
}
func TestTryCreateRolloutSkipsArchivedProject(t *testing.T) {
@@ -204,7 +204,7 @@ func TestTryCreateRolloutSkipsArchivedProject(t *testing.T) {
tasks, err := s.ListTasks(ctx, &store.TaskFind{ProjectID: plan.ProjectID, PlanID: &plan.UID})
require.NoError(t, err)
require.Empty(t, tasks)
- require.Empty(t, b.TaskRunTickleChan)
+ require.Empty(t, b.TaskRunPendingTickleChan)
}
func TestTryCreateRolloutResolvesProjectInstanceOutsideRequestContext(t *testing.T) {
@@ -277,7 +277,7 @@ func TestTryCreateRolloutResolvesProjectInstanceOutsideRequestContext(t *testing
tasks, err := s.ListTasks(ctx, &store.TaskFind{ProjectID: projectID, PlanID: &plan.UID})
require.NoError(t, err)
require.Len(t, tasks, 1)
- require.Len(t, b.TaskRunTickleChan, 1)
+ require.Len(t, b.TaskRunPendingTickleChan, 1)
}
func TestTryCreateRolloutRejectsArchivedProjectInstanceWithWarmDatabaseCache(t *testing.T) {
@@ -367,7 +367,7 @@ func TestTryCreateRolloutRejectsArchivedProjectInstanceWithWarmDatabaseCache(t *
tasks, err := s.ListTasks(ctx, &store.TaskFind{ProjectID: projectID, PlanID: &plan.UID})
require.NoError(t, err)
require.Empty(t, tasks)
- require.Empty(t, b.TaskRunTickleChan)
+ require.Empty(t, b.TaskRunPendingTickleChan)
}
func setupRolloutCreatorStore(ctx context.Context, t *testing.T) *store.Store {
diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.go
index d49b3b893f..15c7741411 100644
--- a/backend/runner/taskrun/running_scheduler.go
+++ b/backend/runner/taskrun/running_scheduler.go
@@ -38,7 +38,7 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa
if err := s.scheduleRunningTaskRuns(ctx); err != nil {
slog.Error("failed to schedule running task runs", log.BBError(err))
}
- case <-s.bus.TaskRunTickleChan:
+ case <-s.bus.TaskRunRunningTickleChan:
if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {
continue
}