Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: serialize rollout run and skip (#20935)

p0nya430918c28d0
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.go
index a7ff01c02d..034a167108 100644
--- a/backend/api/v1/rollout_service.go
+++ b/backend/api/v1/rollout_service.go
@@ -1023,6 +1023,9 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Reques
}
if err := s.store.BatchSkipTasks(ctx, projectID, taskUIDs, request.Reason); err != nil {
+ if common.ErrorCode(err) == common.Conflict {
+ return nil, connect.NewError(connect.CodeFailedPrecondition, err)
+ }
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to skip tasks"))
}
diff --git a/backend/store/task.go b/backend/store/task.go
index e04da3ff1b..9cd1c08bb8 100644
--- a/backend/store/task.go
+++ b/backend/store/task.go
@@ -479,26 +479,84 @@ func (s *Store) ListTaskStatusCountByPlanIDs(ctx context.Context, projectIDs []s
// BatchSkipTasks batch skip tasks.
func (s *Store) BatchSkipTasks(ctx context.Context, projectID string, taskUIDs []int64, comment string) error {
+ if len(taskUIDs) == 0 {
+ return nil
+ }
+
+ tx, err := s.GetDB().BeginTx(ctx, nil)
+ if err != nil {
+ return errors.Wrapf(err, "failed to begin tx")
+ }
+ defer tx.Rollback()
+
+ // Lock task rows before checking task runs so the check uses a fresh Read Committed snapshot.
+ lockQ := qb.Q().Space(`
+ SELECT id
+ FROM task
+ WHERE project = ? AND id = ANY(?)
+ ORDER BY id
+ FOR UPDATE`, projectID, taskUIDs)
+ lockQuery, lockArgs, err := lockQ.ToSQL()
+ if err != nil {
+ return errors.Wrapf(err, "failed to build task lock sql")
+ }
+ if err := func() error {
+ rows, err := tx.QueryContext(ctx, lockQuery, lockArgs...)
+ if err != nil {
+ return errors.Wrapf(err, "failed to lock tasks")
+ }
+ defer rows.Close()
+ for rows.Next() {
+ var taskUID int64
+ if err := rows.Scan(&taskUID); err != nil {
+ return errors.Wrapf(err, "failed to scan locked task")
+ }
+ }
+ if err := rows.Err(); err != nil {
+ return errors.Wrapf(err, "failed to read locked tasks")
+ }
+ return nil
+ }(); err != nil {
+ return err
+ }
+
+ blockingQ := qb.Q().Space(`
+ SELECT task_id
+ FROM task_run
+ WHERE project = ? AND task_id = ANY(?)
+ AND status IN (?, ?, ?, ?)
+ ORDER BY task_id
+ LIMIT 1`, projectID, taskUIDs,
+ storepb.TaskRun_PENDING.String(), storepb.TaskRun_AVAILABLE.String(), storepb.TaskRun_RUNNING.String(), storepb.TaskRun_DONE.String())
+ blockingQuery, blockingArgs, err := blockingQ.ToSQL()
+ if err != nil {
+ return errors.Wrapf(err, "failed to build task run check sql")
+ }
+ var blockingTaskUID int64
+ if err := tx.QueryRowContext(ctx, blockingQuery, blockingArgs...).Scan(&blockingTaskUID); err != nil && err != sql.ErrNoRows {
+ return errors.Wrapf(err, "failed to check task runs")
+ } else if err == nil {
+ return common.Errorf(common.Conflict, "task %d cannot be skipped because it has a task run in a blocking status", blockingTaskUID)
+ }
+
q := qb.Q().Space(`
UPDATE task AS t
SET payload = payload || jsonb_build_object('skipped', ?::BOOLEAN) || jsonb_build_object('skippedReason', ?::TEXT)
WHERE t.id = ANY(?) AND t.project = ?
- AND (t.payload->>'skipped')::BOOLEAN IS NOT TRUE
- AND NOT EXISTS (
- SELECT 1 FROM task_run
- WHERE task_run.task_id = t.id AND task_run.project = t.project
- AND task_run.status IN (?, ?, ?, ?)
- )`, true, comment, taskUIDs, projectID,
- storepb.TaskRun_PENDING.String(), storepb.TaskRun_AVAILABLE.String(), storepb.TaskRun_RUNNING.String(), storepb.TaskRun_DONE.String())
+ AND (t.payload->>'skipped')::BOOLEAN IS NOT TRUE`, true, comment, taskUIDs, projectID)
query, args, err := q.ToSQL()
if err != nil {
return errors.Wrapf(err, "failed to build sql")
}
- if _, err := s.GetDB().ExecContext(ctx, query, args...); err != nil {
+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {
return errors.Wrapf(err, "failed to batch skip tasks")
}
+ if err := tx.Commit(); err != nil {
+ return errors.Wrapf(err, "failed to commit tx")
+ }
+
return nil
}
diff --git a/backend/tests/task_run_idempotent_test.go b/backend/tests/task_run_idempotent_test.go
index 6523bc1c1a..fd89910e35 100644
--- a/backend/tests/task_run_idempotent_test.go
+++ b/backend/tests/task_run_idempotent_test.go
@@ -2,6 +2,7 @@ package tests
import (
"context"
+ "database/sql"
"fmt"
"testing"
"time"
@@ -161,6 +162,161 @@ func TestBatchRunTasks_Idempotent(t *testing.T) {
a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(err))
}
+func TestBatchRunTasks_ConcurrentSkipCannotBothWin(t *testing.T) {
+ a := require.New(t)
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ ctl := &controller{}
+ ctx, err := ctl.StartServerWithExternalPg(ctx)
+ a.NoError(err)
+ defer ctl.Close(ctx)
+
+ instanceDir, err := ctl.provisionSQLiteInstance(t.TempDir(), "testInstanceConcurrentRunSkip")
+ a.NoError(err)
+ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{
+ InstanceId: generateRandomString("instance"),
+ Instance: &v1pb.Instance{
+ Title: "testInstanceConcurrentRunSkip",
+ Engine: v1pb.Engine_SQLITE,
+ Environment: new("environments/prod"),
+ Activation: true,
+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: instanceDir, Id: "admin"}},
+ },
+ }))
+ a.NoError(err)
+
+ const databaseName = "testConcurrentRunSkipDb"
+ a.NoError(ctl.createDatabase(ctx, ctl.project, instanceResp.Msg, nil /* environment */, databaseName, ""))
+ databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{
+ Name: fmt.Sprintf("%s/databases/%s", instanceResp.Msg.Name, databaseName),
+ }))
+ a.NoError(err)
+ sheetResp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{
+ Parent: ctl.project.Name,
+ Sheet: &v1pb.Sheet{Content: []byte("SELECT 1;")},
+ }))
+ a.NoError(err)
+ planResp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{
+ Parent: ctl.project.Name,
+ Plan: &v1pb.Plan{Specs: []*v1pb.Plan_Spec{{
+ Id: uuid.NewString(),
+ Config: &v1pb.Plan_Spec_ChangeDatabaseConfig{
+ ChangeDatabaseConfig: &v1pb.Plan_ChangeDatabaseConfig{
+ Targets: []string{databaseResp.Msg.Name},
+ Sheet: sheetResp.Msg.Name,
+ },
+ },
+ }}},
+ }))
+ a.NoError(err)
+ rolloutResp, err := ctl.rolloutServiceClient.CreateRollout(ctx, connect.NewRequest(&v1pb.CreateRolloutRequest{
+ Parent: planResp.Msg.Name,
+ }))
+ a.NoError(err)
+ stage := rolloutResp.Msg.Stages[0]
+ task := stage.Tasks[0]
+
+ db, err := sql.Open("pgx", ctl.profile.PgURL)
+ a.NoError(err)
+ defer db.Close()
+ lockConn, err := db.Conn(ctx)
+ a.NoError(err)
+ defer lockConn.Close()
+
+ const advisoryLockID = 9900
+ _, err = lockConn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID)
+ a.NoError(err)
+ lockReleased := false
+ defer func() {
+ if !lockReleased {
+ _, _ = lockConn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID)
+ }
+ }()
+ _, err = db.ExecContext(ctx, `
+ CREATE FUNCTION block_task_run_insert() RETURNS trigger AS $$
+ BEGIN
+ PERFORM pg_advisory_xact_lock(9900);
+ RETURN NEW;
+ END;
+ $$ LANGUAGE plpgsql;
+ CREATE TRIGGER block_task_run_insert
+ AFTER INSERT ON task_run
+ FOR EACH ROW EXECUTE FUNCTION block_task_run_insert();
+ `)
+ a.NoError(err)
+
+ runResult := make(chan error, 1)
+ go func() {
+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{
+ Parent: stage.Name,
+ Tasks: []string{task.Name},
+ RunTime: timestamppb.New(time.Now().Add(time.Hour)),
+ }))
+ runResult <- err
+ }()
+ a.Eventually(func() bool {
+ var waiting bool
+ err := db.QueryRowContext(ctx, `
+ SELECT EXISTS (
+ SELECT 1
+ FROM pg_locks
+ WHERE locktype = 'advisory'
+ AND objid = $1
+ AND NOT granted
+ )
+ `, advisoryLockID).Scan(&waiting)
+ return err == nil && waiting
+ }, 10*time.Second, 10*time.Millisecond, "run request should reach the blocked task_run insert")
+
+ skipResult := make(chan error, 1)
+ go func() {
+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{
+ Parent: stage.Name,
+ Tasks: []string{task.Name},
+ Reason: "concurrent skip",
+ }))
+ skipResult <- err
+ }()
+ a.Eventually(func() bool {
+ var waiting bool
+ err := db.QueryRowContext(ctx, `
+ WITH run_backend AS (
+ SELECT pid
+ FROM pg_locks
+ WHERE locktype = 'advisory'
+ AND objid = $1
+ AND NOT granted
+ LIMIT 1
+ )
+ SELECT EXISTS (
+ SELECT 1
+ FROM pg_stat_activity AS activity, run_backend
+ WHERE run_backend.pid = ANY(pg_blocking_pids(activity.pid))
+ )
+ `, advisoryLockID).Scan(&waiting)
+ return err == nil && waiting
+ }, 10*time.Second, 10*time.Millisecond, "skip request should block behind run creation's task lock")
+
+ _, err = lockConn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", advisoryLockID)
+ a.NoError(err)
+ lockReleased = true
+ a.NoError(<-runResult)
+ skipErr := <-skipResult
+ a.Error(skipErr)
+ a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(skipErr))
+
+ rolloutAfter, err := ctl.rolloutServiceClient.GetRollout(ctx, connect.NewRequest(&v1pb.GetRolloutRequest{
+ Name: rolloutResp.Msg.Name,
+ }))
+ a.NoError(err)
+ a.Equal(v1pb.Task_PENDING, rolloutAfter.Msg.Stages[0].Tasks[0].Status)
+ taskRunsResp, err := ctl.rolloutServiceClient.ListTaskRuns(ctx, connect.NewRequest(&v1pb.ListTaskRunsRequest{
+ Parent: task.Name,
+ }))
+ a.NoError(err)
+ a.Len(taskRunsResp.Msg.TaskRuns, 1)
+}
+
func TestBatchRunTasks_RejectsSkippedTasks(t *testing.T) {
a := require.New(t)
ctx := context.Background()