Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: add advisory lock mutex for pending scheduler (#18798)

p0ny6471fa1ec862
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
index 29b559107f..52cf75c1aa 100644
--- a/backend/runner/taskrun/pending_scheduler.go
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -39,6 +39,17 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Wa
}
func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
+ // Acquire cluster-wide mutex - only one replica runs at a time.
+ lock, acquired, err := s.store.TryAdvisoryLock(ctx, store.AdvisoryLockKeyPendingScheduler)
+ if err != nil {
+ return errors.Wrapf(err, "failed to acquire advisory lock")
+ }
+ if !acquired {
+ // Another replica is running, skip this cycle.
+ return nil
+ }
+ defer func() { _ = lock.Release() }()
+
taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
Status: &[]storepb.TaskRun_Status{storepb.TaskRun_PENDING},
})
diff --git a/backend/store/advisory_lock.go b/backend/store/advisory_lock.go
new file mode 100644
index 0000000000..d4b198f07a
--- /dev/null
+++ b/backend/store/advisory_lock.go
@@ -0,0 +1,55 @@
+package store
+
+import (
+ "context"
+ "database/sql"
+)
+
+// AdvisoryLockKey defines lock identifiers for distributed coordination.
+// Each scheduler/component needing cluster-wide mutex gets a unique key.
+type AdvisoryLockKey int64
+
+const (
+ // AdvisoryLockKeyPendingScheduler is used by the pending task run scheduler
+ // to ensure only one replica promotes PENDING → AVAILABLE at a time.
+ AdvisoryLockKeyPendingScheduler AdvisoryLockKey = 1001
+)
+
+// AdvisoryLock holds a dedicated connection for a session-level advisory lock.
+type AdvisoryLock struct {
+ conn *sql.Conn
+ key AdvisoryLockKey
+}
+
+// TryAdvisoryLock attempts to acquire a session-level advisory lock using a
+// dedicated connection. Returns (lock, true) if acquired, (nil, false) if
+// already held by another session. Caller must call lock.Release() when done.
+func (s *Store) TryAdvisoryLock(ctx context.Context, key AdvisoryLockKey) (*AdvisoryLock, bool, error) {
+ conn, err := s.dbConnManager.GetDB().Conn(ctx)
+ if err != nil {
+ return nil, false, err
+ }
+
+ var acquired bool
+ if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", int64(key)).Scan(&acquired); err != nil {
+ conn.Close()
+ return nil, false, err
+ }
+
+ if !acquired {
+ conn.Close()
+ return nil, false, nil
+ }
+
+ return &AdvisoryLock{conn: conn, key: key}, true, nil
+}
+
+// Release releases the advisory lock and returns the connection to the pool.
+func (l *AdvisoryLock) Release() error {
+ if l.conn == nil {
+ return nil
+ }
+ // Unlock then close; closing also releases but explicit unlock is cleaner
+ _, _ = l.conn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", int64(l.key))
+ return l.conn.Close()
+}