Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: make task run creation idempotent to prevent double-click errors (#18183)
Danny Xu42a2f2830010
diff --git a/action/command/rollout.go b/action/command/rollout.goindex a0c31197f1..3ddb1048d7 100644--- a/action/command/rollout.go+++ b/action/command/rollout.go@@ -358,10 +358,7 @@ func waitForRollout(ctx context.Context, w *world.World, client *Client, pendingParent: stage.Name,Tasks: notStartedTasks,}); err != nil {- // Check for specific error indicating task runs already exist (retryable)- if !strings.Contains(err.Error(), "cannot create pending task runs because there are pending/running/done task runs") {- return errors.Wrapf(err, "failed to batch create tasks")- }+ return errors.Wrapf(err, "failed to batch create tasks")}}time.Sleep(5 * time.Second)diff --git a/backend/store/task_run.go b/backend/store/task_run.goindex 3ba62f6ebd..25524a150f 100644--- a/backend/store/task_run.go+++ b/backend/store/task_run.go@@ -3,7 +3,6 @@ package storeimport ("context""database/sql"- "slices""time""github.com/pkg/errors"@@ -258,90 +257,15 @@ func (s *Store) UpdateTaskRunStartAt(ctx context.Context, taskRunID int) error {}// CreatePendingTaskRuns creates pending task runs.+// This operation is idempotent and safe for concurrent calls:+// - Uses WHERE NOT EXISTS to skip tasks that already have active (PENDING/RUNNING/DONE) task runs+// - Uses ON CONFLICT DO NOTHING to handle race conditions where two requests try to create the same task run+// - The unique constraint on (task_id, attempt) ensures no duplicatesfunc (s *Store) CreatePendingTaskRuns(ctx context.Context, creatorID int, creates ...*TaskRunMessage) error {if len(creates) == 0 {return nil}- slices.SortFunc(creates, func(a, b *TaskRunMessage) int {- return a.TaskUID - b.TaskUID- })-- var taskIDs []int- for _, create := range creates {- taskIDs = append(taskIDs, create.TaskUID)- }-- tx, err := s.GetDB().BeginTx(ctx, nil)- if err != nil {- return errors.Wrapf(err, "failed to begin tx")- }- defer tx.Rollback()-- attempts, err := s.getTaskNextAttempt(ctx, tx, taskIDs)- if err != nil {- return errors.Wrapf(err, "failed to get task next attempt")- }-- exist, err := s.checkTaskRunsExist(ctx, tx, taskIDs, []storepb.TaskRun_Status{storepb.TaskRun_PENDING, storepb.TaskRun_RUNNING, storepb.TaskRun_DONE})- if err != nil {- return errors.Wrapf(err, "failed to check if task runs exist")- }- if exist {- return errors.Errorf("cannot create pending task runs because there are pending/running/done task runs")- }-- if err := s.createPendingTaskRunsTx(ctx, tx, creatorID, attempts, creates); err != nil {- return errors.Wrapf(err, "failed to create pending task runs")- }-- if err := tx.Commit(); err != nil {- return errors.Wrapf(err, "failed to commit tx")- }-- return nil-}--func (*Store) getTaskNextAttempt(ctx context.Context, txn *sql.Tx, taskIDs []int) ([]int, error) {- q := qb.Q().Space(`- WITH tasks AS (- SELECT id FROM unnest(CAST(? AS INTEGER[])) AS id- )- SELECT- (SELECT COALESCE(MAX(attempt)+1, 0) FROM task_run WHERE task_run.task_id = tasks.id)- FROM tasks ORDER BY tasks.id ASC- `, taskIDs)-- query, args, err := q.ToSQL()- if err != nil {- return nil, errors.Wrapf(err, "failed to build sql")- }-- rows, err := txn.QueryContext(ctx, query, args...)- if err != nil {- return nil, errors.Wrap(err, "failed to query")- }- defer rows.Close()-- var attempts []int- for rows.Next() {- var attempt int- if err := rows.Scan(&attempt); err != nil {- return nil, errors.Wrap(err, "failed to scan")- }- attempts = append(attempts, attempt)- }- if err := rows.Err(); err != nil {- return nil, errors.Wrap(err, "failed to scan rows")- }-- return attempts, nil-}--func (*Store) createPendingTaskRunsTx(ctx context.Context, txn *sql.Tx, creatorID int, attempts []int, creates []*TaskRunMessage) error {- if len(attempts) != len(creates) {- return errors.Errorf("length of attempts and creates are different")- }var taskUIDs []intvar sheetUIDs []*intvar runAts []*time.Time@@ -351,6 +275,11 @@ func (*Store) createPendingTaskRunsTx(ctx context.Context, txn *sql.Tx, creatorIrunAts = append(runAts, create.RunAt)}+ // Single query that:+ // 1. Filters out tasks with existing PENDING/RUNNING/DONE task runs (idempotent)+ // 2. Calculates next attempt for each remaining task+ // 3. Inserts task runs+ // 4. Uses ON CONFLICT DO NOTHING to handle race conditionsq := qb.Q().Space(`INSERT INTO task_run (creator_id,@@ -359,51 +288,39 @@ func (*Store) createPendingTaskRunsTx(ctx context.Context, txn *sql.Tx, creatorIrun_at,attempt,status- ) SELECT+ )+ SELECT?,- unnest(CAST(? AS INTEGER[])),- unnest(CAST(? AS INTEGER[])),- unnest(CAST(? AS TIMESTAMPTZ[])),- unnest(CAST(? AS INTEGER[])),+ tasks.task_id,+ tasks.sheet_id,+ tasks.run_at,+ COALESCE((SELECT MAX(attempt) + 1 FROM task_run WHERE task_run.task_id = tasks.task_id), 0) as attempt,?- `, creatorID, taskUIDs, sheetUIDs, runAts, attempts, storepb.TaskRun_PENDING.String())+ FROM (+ SELECT+ unnest(CAST(? AS INTEGER[])) AS task_id,+ unnest(CAST(? AS INTEGER[])) AS sheet_id,+ unnest(CAST(? AS TIMESTAMPTZ[])) AS run_at+ ) tasks+ WHERE NOT EXISTS (+ SELECT 1 FROM task_run+ WHERE task_run.task_id = tasks.task_id+ AND task_run.status IN (?, ?, ?)+ )+ ON CONFLICT (task_id, attempt) DO NOTHING+ `, creatorID, storepb.TaskRun_PENDING.String(), taskUIDs, sheetUIDs, runAts,+ storepb.TaskRun_PENDING.String(), storepb.TaskRun_RUNNING.String(), storepb.TaskRun_DONE.String())query, args, err := q.ToSQL()if err != nil {return errors.Wrapf(err, "failed to build sql")}- if _, err := txn.ExecContext(ctx, query, args...); err != nil {+ if _, err := s.GetDB().ExecContext(ctx, query, args...); err != nil {return errors.Wrapf(err, "failed to create pending task runs")}- return nil-}--func (*Store) checkTaskRunsExist(ctx context.Context, txn *sql.Tx, taskIDs []int, statuses []storepb.TaskRun_Status) (bool, error) {- var statusStrings []string- for _, status := range statuses {- statusStrings = append(statusStrings, status.String())- }-- q := qb.Q().Space(`- SELECT EXISTS (- SELECT 1- FROM task_run- WHERE task_run.task_id = ANY(?) AND task_run.status = ANY(?)- )- `, taskIDs, statusStrings)-- query, args, err := q.ToSQL()- if err != nil {- return false, errors.Wrapf(err, "failed to build sql")- }-- var exist bool- if err := txn.QueryRowContext(ctx, query, args...).Scan(&exist); err != nil {- return false, errors.Wrapf(err, "failed to query if task runs exist")- }- return exist, nil+ return nil}// patchTaskRunStatusImpl updates a taskRun status. Returns the new state of the taskRun after update.diff --git a/backend/tests/task_run_idempotent_test.go b/backend/tests/task_run_idempotent_test.gonew file mode 100644index 0000000000..d5159c10d3--- /dev/null+++ b/backend/tests/task_run_idempotent_test.go@@ -0,0 +1,152 @@+package tests++import (+ "context"+ "fmt"+ "testing"++ "connectrpc.com/connect"+ "github.com/google/uuid"+ "github.com/stretchr/testify/require"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// TestBatchRunTasks_Idempotent verifies that BatchRunTasks is idempotent and safe for concurrent calls.+// This test covers both sequential double-clicks and concurrent requests scenarios.+func TestBatchRunTasks_Idempotent(t *testing.T) {+ a := require.New(t)+ ctx := context.Background()+ ctl := &controller{}+ ctx, err := ctl.StartServerWithExternalPg(ctx)+ a.NoError(err)+ defer ctl.Close(ctx)++ // Provision an instance+ instanceRootDir := t.TempDir()+ instanceName := "testInstanceIdempotent"+ instanceDir, err := ctl.provisionSQLiteInstance(instanceRootDir, instanceName)+ a.NoError(err)++ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{+ InstanceId: generateRandomString("instance"),+ Instance: &v1pb.Instance{+ Title: instanceName,+ Engine: v1pb.Engine_SQLITE,+ Environment: stringPtr("environments/prod"),+ Activation: true,+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: instanceDir, Id: "admin"}},+ },+ }))+ a.NoError(err)+ instance := instanceResp.Msg++ // Create a database+ databaseName := "testIdempotentDb"+ err = ctl.createDatabaseV2(ctx, ctl.project, instance, nil /* environment */, databaseName, "")+ a.NoError(err)++ databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{+ Name: fmt.Sprintf("%s/databases/%s", instance.Name, databaseName),+ }))+ a.NoError(err)+ database := databaseResp.Msg++ // Create a sheet with SQL+ sheetResp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{+ Parent: ctl.project.Name,+ Sheet: &v1pb.Sheet{+ Title: "Test Sheet",+ Content: []byte("SELECT 1;"),+ },+ }))+ a.NoError(err)+ sheet := sheetResp.Msg++ // Create a plan with change database config+ spec := &v1pb.Plan_Spec{+ Id: uuid.NewString(),+ Config: &v1pb.Plan_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &v1pb.Plan_ChangeDatabaseConfig{+ Targets: []string{database.Name},+ Sheet: sheet.Name,+ Type: v1pb.DatabaseChangeType_MIGRATE,+ },+ },+ }++ planResp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{+ Parent: ctl.project.Name,+ Plan: &v1pb.Plan{+ Specs: []*v1pb.Plan_Spec{spec},+ },+ }))+ a.NoError(err)+ plan := planResp.Msg++ // Create a rollout+ rolloutResp, err := ctl.rolloutServiceClient.CreateRollout(ctx, connect.NewRequest(&v1pb.CreateRolloutRequest{+ Parent: ctl.project.Name,+ Rollout: &v1pb.Rollout{Plan: plan.Name},+ }))+ a.NoError(err)+ rollout := rolloutResp.Msg++ // Get the tasks from the rollout+ a.GreaterOrEqual(len(rollout.Stages), 1, "rollout should have at least one stage")+ stage := rollout.Stages[0]+ a.GreaterOrEqual(len(stage.Tasks), 1, "stage should have at least one task")++ var taskNames []string+ for _, task := range stage.Tasks {+ taskNames = append(taskNames, task.Name)+ }++ // Test 1: Concurrent calls (hardest case - simulates rapid multiple clicks)+ // Make 5 concurrent BatchRunTasks calls+ type result struct {+ err error+ }+ results := make(chan result, 5)++ for range 5 {+ go func() {+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{+ Parent: stage.Name,+ Tasks: taskNames,+ }))+ results <- result{err: err}+ }()+ }++ // Collect results - all should succeed+ for i := range 5 {+ res := <-results+ a.NoError(res.err, "concurrent BatchRunTasks call %d should succeed", i+1)+ }++ // Verify no duplicate task runs were created despite concurrent requests+ for _, taskName := range taskNames {+ taskRunsResp, err := ctl.rolloutServiceClient.ListTaskRuns(ctx, connect.NewRequest(&v1pb.ListTaskRunsRequest{+ Parent: taskName,+ }))+ a.NoError(err)+ a.Equal(1, len(taskRunsResp.Msg.TaskRuns), "task %s should have exactly one task run after concurrent calls", taskName)+ }++ // Test 2: Sequential double-click (simulates user accidentally clicking twice)+ _, err = ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{+ Parent: stage.Name,+ Tasks: taskNames,+ }))+ a.NoError(err, "sequential BatchRunTasks call should succeed")++ // Verify still exactly one task run per task+ for _, taskName := range taskNames {+ taskRunsResp, err := ctl.rolloutServiceClient.ListTaskRuns(ctx, connect.NewRequest(&v1pb.ListTaskRunsRequest{+ Parent: taskName,+ }))+ a.NoError(err)+ a.Equal(1, len(taskRunsResp.Msg.TaskRuns), "task %s should still have exactly one task run after sequential call", taskName)+ }+}