Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: serialize rollout run and skip (#20935)
p0nya430918c28d0
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.goindex a7ff01c02d..034a167108 100644--- a/backend/api/v1/rollout_service.go+++ b/backend/api/v1/rollout_service.go@@ -1023,6 +1023,9 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Reques}if err := s.store.BatchSkipTasks(ctx, projectID, taskUIDs, request.Reason); err != nil {+ if common.ErrorCode(err) == common.Conflict {+ return nil, connect.NewError(connect.CodeFailedPrecondition, err)+ }return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to skip tasks"))}diff --git a/backend/store/task.go b/backend/store/task.goindex e04da3ff1b..9cd1c08bb8 100644--- a/backend/store/task.go+++ b/backend/store/task.go@@ -479,26 +479,84 @@ func (s *Store) ListTaskStatusCountByPlanIDs(ctx context.Context, projectIDs []s// BatchSkipTasks batch skip tasks.func (s *Store) BatchSkipTasks(ctx context.Context, projectID string, taskUIDs []int64, comment string) error {+ if len(taskUIDs) == 0 {+ return nil+ }++ tx, err := s.GetDB().BeginTx(ctx, nil)+ if err != nil {+ return errors.Wrapf(err, "failed to begin tx")+ }+ defer tx.Rollback()++ // Lock task rows before checking task runs so the check uses a fresh Read Committed snapshot.+ lockQ := qb.Q().Space(`+ SELECT id+ FROM task+ WHERE project = ? AND id = ANY(?)+ ORDER BY id+ FOR UPDATE`, projectID, taskUIDs)+ lockQuery, lockArgs, err := lockQ.ToSQL()+ if err != nil {+ return errors.Wrapf(err, "failed to build task lock sql")+ }+ if err := func() error {+ rows, err := tx.QueryContext(ctx, lockQuery, lockArgs...)+ if err != nil {+ return errors.Wrapf(err, "failed to lock tasks")+ }+ defer rows.Close()+ for rows.Next() {+ var taskUID int64+ if err := rows.Scan(&taskUID); err != nil {+ return errors.Wrapf(err, "failed to scan locked task")+ }+ }+ if err := rows.Err(); err != nil {+ return errors.Wrapf(err, "failed to read locked tasks")+ }+ return nil+ }(); err != nil {+ return err+ }++ blockingQ := qb.Q().Space(`+ SELECT task_id+ FROM task_run+ WHERE project = ? AND task_id = ANY(?)+ AND status IN (?, ?, ?, ?)+ ORDER BY task_id+ LIMIT 1`, projectID, taskUIDs,+ storepb.TaskRun_PENDING.String(), storepb.TaskRun_AVAILABLE.String(), storepb.TaskRun_RUNNING.String(), storepb.TaskRun_DONE.String())+ blockingQuery, blockingArgs, err := blockingQ.ToSQL()+ if err != nil {+ return errors.Wrapf(err, "failed to build task run check sql")+ }+ var blockingTaskUID int64+ if err := tx.QueryRowContext(ctx, blockingQuery, blockingArgs...).Scan(&blockingTaskUID); err != nil && err != sql.ErrNoRows {+ return errors.Wrapf(err, "failed to check task runs")+ } else if err == nil {+ return common.Errorf(common.Conflict, "task %d cannot be skipped because it has a task run in a blocking status", blockingTaskUID)+ }+q := qb.Q().Space(`UPDATE task AS tSET payload = payload || jsonb_build_object('skipped', ?::BOOLEAN) || jsonb_build_object('skippedReason', ?::TEXT)WHERE t.id = ANY(?) AND t.project = ?- AND (t.payload->>'skipped')::BOOLEAN IS NOT TRUE- AND NOT EXISTS (- SELECT 1 FROM task_run- WHERE task_run.task_id = t.id AND task_run.project = t.project- AND task_run.status IN (?, ?, ?, ?)- )`, true, comment, taskUIDs, projectID,- storepb.TaskRun_PENDING.String(), storepb.TaskRun_AVAILABLE.String(), storepb.TaskRun_RUNNING.String(), storepb.TaskRun_DONE.String())+ AND (t.payload->>'skipped')::BOOLEAN IS NOT TRUE`, true, comment, taskUIDs, projectID)query, args, err := q.ToSQL()if err != nil {return errors.Wrapf(err, "failed to build sql")}- if _, err := s.GetDB().ExecContext(ctx, query, args...); err != nil {+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {return errors.Wrapf(err, "failed to batch skip tasks")}+ if err := tx.Commit(); err != nil {+ return errors.Wrapf(err, "failed to commit tx")+ }+return nil}diff --git a/backend/tests/task_run_idempotent_test.go b/backend/tests/task_run_idempotent_test.goindex 6523bc1c1a..fd89910e35 100644--- a/backend/tests/task_run_idempotent_test.go+++ b/backend/tests/task_run_idempotent_test.go@@ -2,6 +2,7 @@ package testsimport ("context"+ "database/sql""fmt""testing""time"@@ -161,6 +162,161 @@ func TestBatchRunTasks_Idempotent(t *testing.T) {a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(err))}+func TestBatchRunTasks_ConcurrentSkipCannotBothWin(t *testing.T) {+ a := require.New(t)+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)+ defer cancel()+ ctl := &controller{}+ ctx, err := ctl.StartServerWithExternalPg(ctx)+ a.NoError(err)+ defer ctl.Close(ctx)++ instanceDir, err := ctl.provisionSQLiteInstance(t.TempDir(), "testInstanceConcurrentRunSkip")+ a.NoError(err)+ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{+ InstanceId: generateRandomString("instance"),+ Instance: &v1pb.Instance{+ Title: "testInstanceConcurrentRunSkip",+ Engine: v1pb.Engine_SQLITE,+ Environment: new("environments/prod"),+ Activation: true,+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: instanceDir, Id: "admin"}},+ },+ }))+ a.NoError(err)++ const databaseName = "testConcurrentRunSkipDb"+ a.NoError(ctl.createDatabase(ctx, ctl.project, instanceResp.Msg, nil /* environment */, databaseName, ""))+ databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{+ Name: fmt.Sprintf("%s/databases/%s", instanceResp.Msg.Name, databaseName),+ }))+ a.NoError(err)+ sheetResp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{+ Parent: ctl.project.Name,+ Sheet: &v1pb.Sheet{Content: []byte("SELECT 1;")},+ }))+ a.NoError(err)+ planResp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{+ Parent: ctl.project.Name,+ Plan: &v1pb.Plan{Specs: []*v1pb.Plan_Spec{{+ Id: uuid.NewString(),+ Config: &v1pb.Plan_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &v1pb.Plan_ChangeDatabaseConfig{+ Targets: []string{databaseResp.Msg.Name},+ Sheet: sheetResp.Msg.Name,+ },+ },+ }}},+ }))+ a.NoError(err)+ rolloutResp, err := ctl.rolloutServiceClient.CreateRollout(ctx, connect.NewRequest(&v1pb.CreateRolloutRequest{+ Parent: planResp.Msg.Name,+ }))+ a.NoError(err)+ stage := rolloutResp.Msg.Stages[0]+ task := stage.Tasks[0]++ db, err := sql.Open("pgx", ctl.profile.PgURL)+ a.NoError(err)+ defer db.Close()+ lockConn, err := db.Conn(ctx)+ a.NoError(err)+ defer lockConn.Close()++ const advisoryLockID = 9900+ _, err = lockConn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID)+ a.NoError(err)+ lockReleased := false+ defer func() {+ if !lockReleased {+ _, _ = lockConn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID)+ }+ }()+ _, err = db.ExecContext(ctx, `+ CREATE FUNCTION block_task_run_insert() RETURNS trigger AS $$+ BEGIN+ PERFORM pg_advisory_xact_lock(9900);+ RETURN NEW;+ END;+ $$ LANGUAGE plpgsql;+ CREATE TRIGGER block_task_run_insert+ AFTER INSERT ON task_run+ FOR EACH ROW EXECUTE FUNCTION block_task_run_insert();+ `)+ a.NoError(err)++ runResult := make(chan error, 1)+ go func() {+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{+ Parent: stage.Name,+ Tasks: []string{task.Name},+ RunTime: timestamppb.New(time.Now().Add(time.Hour)),+ }))+ runResult <- err+ }()+ a.Eventually(func() bool {+ var waiting bool+ err := db.QueryRowContext(ctx, `+ SELECT EXISTS (+ SELECT 1+ FROM pg_locks+ WHERE locktype = 'advisory'+ AND objid = $1+ AND NOT granted+ )+ `, advisoryLockID).Scan(&waiting)+ return err == nil && waiting+ }, 10*time.Second, 10*time.Millisecond, "run request should reach the blocked task_run insert")++ skipResult := make(chan error, 1)+ go func() {+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{+ Parent: stage.Name,+ Tasks: []string{task.Name},+ Reason: "concurrent skip",+ }))+ skipResult <- err+ }()+ a.Eventually(func() bool {+ var waiting bool+ err := db.QueryRowContext(ctx, `+ WITH run_backend AS (+ SELECT pid+ FROM pg_locks+ WHERE locktype = 'advisory'+ AND objid = $1+ AND NOT granted+ LIMIT 1+ )+ SELECT EXISTS (+ SELECT 1+ FROM pg_stat_activity AS activity, run_backend+ WHERE run_backend.pid = ANY(pg_blocking_pids(activity.pid))+ )+ `, advisoryLockID).Scan(&waiting)+ return err == nil && waiting+ }, 10*time.Second, 10*time.Millisecond, "skip request should block behind run creation's task lock")++ _, err = lockConn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", advisoryLockID)+ a.NoError(err)+ lockReleased = true+ a.NoError(<-runResult)+ skipErr := <-skipResult+ a.Error(skipErr)+ a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(skipErr))++ rolloutAfter, err := ctl.rolloutServiceClient.GetRollout(ctx, connect.NewRequest(&v1pb.GetRolloutRequest{+ Name: rolloutResp.Msg.Name,+ }))+ a.NoError(err)+ a.Equal(v1pb.Task_PENDING, rolloutAfter.Msg.Stages[0].Tasks[0].Status)+ taskRunsResp, err := ctl.rolloutServiceClient.ListTaskRuns(ctx, connect.NewRequest(&v1pb.ListTaskRunsRequest{+ Parent: task.Name,+ }))+ a.NoError(err)+ a.Len(taskRunsResp.Msg.TaskRuns, 1)+}+func TestBatchRunTasks_RejectsSkippedTasks(t *testing.T) {a := require.New(t)ctx := context.Background()