Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: make task run creation idempotent to prevent double-click errors (#18183)

Danny Xu42a2f2830010
diff --git a/action/command/rollout.go b/action/command/rollout.go
index a0c31197f1..3ddb1048d7 100644
--- a/action/command/rollout.go
+++ b/action/command/rollout.go
@@ -358,10 +358,7 @@ func waitForRollout(ctx context.Context, w *world.World, client *Client, pending
Parent: stage.Name,
Tasks: notStartedTasks,
}); err != nil {
- // Check for specific error indicating task runs already exist (retryable)
- if !strings.Contains(err.Error(), "cannot create pending task runs because there are pending/running/done task runs") {
- return errors.Wrapf(err, "failed to batch create tasks")
- }
+ return errors.Wrapf(err, "failed to batch create tasks")
}
}
time.Sleep(5 * time.Second)
diff --git a/backend/store/task_run.go b/backend/store/task_run.go
index 3ba62f6ebd..25524a150f 100644
--- a/backend/store/task_run.go
+++ b/backend/store/task_run.go
@@ -3,7 +3,6 @@ package store
import (
"context"
"database/sql"
- "slices"
"time"
"github.com/pkg/errors"
@@ -258,90 +257,15 @@ func (s *Store) UpdateTaskRunStartAt(ctx context.Context, taskRunID int) error {
}
// CreatePendingTaskRuns creates pending task runs.
+// This operation is idempotent and safe for concurrent calls:
+// - Uses WHERE NOT EXISTS to skip tasks that already have active (PENDING/RUNNING/DONE) task runs
+// - Uses ON CONFLICT DO NOTHING to handle race conditions where two requests try to create the same task run
+// - The unique constraint on (task_id, attempt) ensures no duplicates
func (s *Store) CreatePendingTaskRuns(ctx context.Context, creatorID int, creates ...*TaskRunMessage) error {
if len(creates) == 0 {
return nil
}
- slices.SortFunc(creates, func(a, b *TaskRunMessage) int {
- return a.TaskUID - b.TaskUID
- })
-
- var taskIDs []int
- for _, create := range creates {
- taskIDs = append(taskIDs, create.TaskUID)
- }
-
- tx, err := s.GetDB().BeginTx(ctx, nil)
- if err != nil {
- return errors.Wrapf(err, "failed to begin tx")
- }
- defer tx.Rollback()
-
- attempts, err := s.getTaskNextAttempt(ctx, tx, taskIDs)
- if err != nil {
- return errors.Wrapf(err, "failed to get task next attempt")
- }
-
- exist, err := s.checkTaskRunsExist(ctx, tx, taskIDs, []storepb.TaskRun_Status{storepb.TaskRun_PENDING, storepb.TaskRun_RUNNING, storepb.TaskRun_DONE})
- if err != nil {
- return errors.Wrapf(err, "failed to check if task runs exist")
- }
- if exist {
- return errors.Errorf("cannot create pending task runs because there are pending/running/done task runs")
- }
-
- if err := s.createPendingTaskRunsTx(ctx, tx, creatorID, attempts, creates); err != nil {
- return errors.Wrapf(err, "failed to create pending task runs")
- }
-
- if err := tx.Commit(); err != nil {
- return errors.Wrapf(err, "failed to commit tx")
- }
-
- return nil
-}
-
-func (*Store) getTaskNextAttempt(ctx context.Context, txn *sql.Tx, taskIDs []int) ([]int, error) {
- q := qb.Q().Space(`
- WITH tasks AS (
- SELECT id FROM unnest(CAST(? AS INTEGER[])) AS id
- )
- SELECT
- (SELECT COALESCE(MAX(attempt)+1, 0) FROM task_run WHERE task_run.task_id = tasks.id)
- FROM tasks ORDER BY tasks.id ASC
- `, taskIDs)
-
- query, args, err := q.ToSQL()
- if err != nil {
- return nil, errors.Wrapf(err, "failed to build sql")
- }
-
- rows, err := txn.QueryContext(ctx, query, args...)
- if err != nil {
- return nil, errors.Wrap(err, "failed to query")
- }
- defer rows.Close()
-
- var attempts []int
- for rows.Next() {
- var attempt int
- if err := rows.Scan(&attempt); err != nil {
- return nil, errors.Wrap(err, "failed to scan")
- }
- attempts = append(attempts, attempt)
- }
- if err := rows.Err(); err != nil {
- return nil, errors.Wrap(err, "failed to scan rows")
- }
-
- return attempts, nil
-}
-
-func (*Store) createPendingTaskRunsTx(ctx context.Context, txn *sql.Tx, creatorID int, attempts []int, creates []*TaskRunMessage) error {
- if len(attempts) != len(creates) {
- return errors.Errorf("length of attempts and creates are different")
- }
var taskUIDs []int
var sheetUIDs []*int
var runAts []*time.Time
@@ -351,6 +275,11 @@ func (*Store) createPendingTaskRunsTx(ctx context.Context, txn *sql.Tx, creatorI
runAts = append(runAts, create.RunAt)
}
+ // Single query that:
+ // 1. Filters out tasks with existing PENDING/RUNNING/DONE task runs (idempotent)
+ // 2. Calculates next attempt for each remaining task
+ // 3. Inserts task runs
+ // 4. Uses ON CONFLICT DO NOTHING to handle race conditions
q := qb.Q().Space(`
INSERT INTO task_run (
creator_id,
@@ -359,51 +288,39 @@ func (*Store) createPendingTaskRunsTx(ctx context.Context, txn *sql.Tx, creatorI
run_at,
attempt,
status
- ) SELECT
+ )
+ SELECT
?,
- unnest(CAST(? AS INTEGER[])),
- unnest(CAST(? AS INTEGER[])),
- unnest(CAST(? AS TIMESTAMPTZ[])),
- unnest(CAST(? AS INTEGER[])),
+ tasks.task_id,
+ tasks.sheet_id,
+ tasks.run_at,
+ COALESCE((SELECT MAX(attempt) + 1 FROM task_run WHERE task_run.task_id = tasks.task_id), 0) as attempt,
?
- `, creatorID, taskUIDs, sheetUIDs, runAts, attempts, storepb.TaskRun_PENDING.String())
+ FROM (
+ SELECT
+ unnest(CAST(? AS INTEGER[])) AS task_id,
+ unnest(CAST(? AS INTEGER[])) AS sheet_id,
+ unnest(CAST(? AS TIMESTAMPTZ[])) AS run_at
+ ) tasks
+ WHERE NOT EXISTS (
+ SELECT 1 FROM task_run
+ WHERE task_run.task_id = tasks.task_id
+ AND task_run.status IN (?, ?, ?)
+ )
+ ON CONFLICT (task_id, attempt) DO NOTHING
+ `, creatorID, storepb.TaskRun_PENDING.String(), taskUIDs, sheetUIDs, runAts,
+ storepb.TaskRun_PENDING.String(), storepb.TaskRun_RUNNING.String(), storepb.TaskRun_DONE.String())
query, args, err := q.ToSQL()
if err != nil {
return errors.Wrapf(err, "failed to build sql")
}
- if _, err := txn.ExecContext(ctx, query, args...); err != nil {
+ if _, err := s.GetDB().ExecContext(ctx, query, args...); err != nil {
return errors.Wrapf(err, "failed to create pending task runs")
}
- return nil
-}
-
-func (*Store) checkTaskRunsExist(ctx context.Context, txn *sql.Tx, taskIDs []int, statuses []storepb.TaskRun_Status) (bool, error) {
- var statusStrings []string
- for _, status := range statuses {
- statusStrings = append(statusStrings, status.String())
- }
-
- q := qb.Q().Space(`
- SELECT EXISTS (
- SELECT 1
- FROM task_run
- WHERE task_run.task_id = ANY(?) AND task_run.status = ANY(?)
- )
- `, taskIDs, statusStrings)
-
- query, args, err := q.ToSQL()
- if err != nil {
- return false, errors.Wrapf(err, "failed to build sql")
- }
-
- var exist bool
- if err := txn.QueryRowContext(ctx, query, args...).Scan(&exist); err != nil {
- return false, errors.Wrapf(err, "failed to query if task runs exist")
- }
- return exist, nil
+ return nil
}
// patchTaskRunStatusImpl updates a taskRun status. Returns the new state of the taskRun after update.
diff --git a/backend/tests/task_run_idempotent_test.go b/backend/tests/task_run_idempotent_test.go
new file mode 100644
index 0000000000..d5159c10d3
--- /dev/null
+++ b/backend/tests/task_run_idempotent_test.go
@@ -0,0 +1,152 @@
+package tests
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "connectrpc.com/connect"
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/require"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// TestBatchRunTasks_Idempotent verifies that BatchRunTasks is idempotent and safe for concurrent calls.
+// This test covers both sequential double-clicks and concurrent requests scenarios.
+func TestBatchRunTasks_Idempotent(t *testing.T) {
+ a := require.New(t)
+ ctx := context.Background()
+ ctl := &controller{}
+ ctx, err := ctl.StartServerWithExternalPg(ctx)
+ a.NoError(err)
+ defer ctl.Close(ctx)
+
+ // Provision an instance
+ instanceRootDir := t.TempDir()
+ instanceName := "testInstanceIdempotent"
+ instanceDir, err := ctl.provisionSQLiteInstance(instanceRootDir, instanceName)
+ a.NoError(err)
+
+ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{
+ InstanceId: generateRandomString("instance"),
+ Instance: &v1pb.Instance{
+ Title: instanceName,
+ Engine: v1pb.Engine_SQLITE,
+ Environment: stringPtr("environments/prod"),
+ Activation: true,
+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: instanceDir, Id: "admin"}},
+ },
+ }))
+ a.NoError(err)
+ instance := instanceResp.Msg
+
+ // Create a database
+ databaseName := "testIdempotentDb"
+ err = ctl.createDatabaseV2(ctx, ctl.project, instance, nil /* environment */, databaseName, "")
+ a.NoError(err)
+
+ databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{
+ Name: fmt.Sprintf("%s/databases/%s", instance.Name, databaseName),
+ }))
+ a.NoError(err)
+ database := databaseResp.Msg
+
+ // Create a sheet with SQL
+ sheetResp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{
+ Parent: ctl.project.Name,
+ Sheet: &v1pb.Sheet{
+ Title: "Test Sheet",
+ Content: []byte("SELECT 1;"),
+ },
+ }))
+ a.NoError(err)
+ sheet := sheetResp.Msg
+
+ // Create a plan with change database config
+ spec := &v1pb.Plan_Spec{
+ Id: uuid.NewString(),
+ Config: &v1pb.Plan_Spec_ChangeDatabaseConfig{
+ ChangeDatabaseConfig: &v1pb.Plan_ChangeDatabaseConfig{
+ Targets: []string{database.Name},
+ Sheet: sheet.Name,
+ Type: v1pb.DatabaseChangeType_MIGRATE,
+ },
+ },
+ }
+
+ planResp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{
+ Parent: ctl.project.Name,
+ Plan: &v1pb.Plan{
+ Specs: []*v1pb.Plan_Spec{spec},
+ },
+ }))
+ a.NoError(err)
+ plan := planResp.Msg
+
+ // Create a rollout
+ rolloutResp, err := ctl.rolloutServiceClient.CreateRollout(ctx, connect.NewRequest(&v1pb.CreateRolloutRequest{
+ Parent: ctl.project.Name,
+ Rollout: &v1pb.Rollout{Plan: plan.Name},
+ }))
+ a.NoError(err)
+ rollout := rolloutResp.Msg
+
+ // Get the tasks from the rollout
+ a.GreaterOrEqual(len(rollout.Stages), 1, "rollout should have at least one stage")
+ stage := rollout.Stages[0]
+ a.GreaterOrEqual(len(stage.Tasks), 1, "stage should have at least one task")
+
+ var taskNames []string
+ for _, task := range stage.Tasks {
+ taskNames = append(taskNames, task.Name)
+ }
+
+ // Test 1: Concurrent calls (hardest case - simulates rapid multiple clicks)
+ // Make 5 concurrent BatchRunTasks calls
+ type result struct {
+ err error
+ }
+ results := make(chan result, 5)
+
+ for range 5 {
+ go func() {
+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{
+ Parent: stage.Name,
+ Tasks: taskNames,
+ }))
+ results <- result{err: err}
+ }()
+ }
+
+ // Collect results - all should succeed
+ for i := range 5 {
+ res := <-results
+ a.NoError(res.err, "concurrent BatchRunTasks call %d should succeed", i+1)
+ }
+
+ // Verify no duplicate task runs were created despite concurrent requests
+ for _, taskName := range taskNames {
+ taskRunsResp, err := ctl.rolloutServiceClient.ListTaskRuns(ctx, connect.NewRequest(&v1pb.ListTaskRunsRequest{
+ Parent: taskName,
+ }))
+ a.NoError(err)
+ a.Equal(1, len(taskRunsResp.Msg.TaskRuns), "task %s should have exactly one task run after concurrent calls", taskName)
+ }
+
+ // Test 2: Sequential double-click (simulates user accidentally clicking twice)
+ _, err = ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{
+ Parent: stage.Name,
+ Tasks: taskNames,
+ }))
+ a.NoError(err, "sequential BatchRunTasks call should succeed")
+
+ // Verify still exactly one task run per task
+ for _, taskName := range taskNames {
+ taskRunsResp, err := ctl.rolloutServiceClient.ListTaskRuns(ctx, connect.NewRequest(&v1pb.ListTaskRunsRequest{
+ Parent: taskName,
+ }))
+ a.NoError(err)
+ a.Equal(1, len(taskRunsResp.Msg.TaskRuns), "task %s should still have exactly one task run after sequential call", taskName)
+ }
+}