Git designed for AI agents.
package datastore
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/feder-cr/git-ai/internal/helper"
"github.com/feder-cr/git-ai/internal/log"
"github.com/feder-cr/git-ai/internal/praefect/config"
"github.com/feder-cr/git-ai/internal/praefect/datastore/glsql"
"github.com/prometheus/client_golang/prometheus"
)
// RepositoryReferenceWriteLockReleasesChannel is the PostgreSQL channel on
// which notify_on_write_lock_release() sends NOTIFY events.
const RepositoryReferenceWriteLockReleasesChannel = "repository_reference_write_lock_releases"
var lockReleasingListenerReconnectsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gitaly_praefect_lock_release_listener_reconnects_total",
Help: "Counts amount of reconnects to listen for notification from write lock release channel",
},
[]string{"state"},
)
// LockID identifies a repository reference write lock, formatted as "virtualStorage|relativePath".
type LockID string
// RepoLock represents an acquired repository write lock. Unlock releases
// the lock; Renew extends its lease and must be called periodically until
// Unlock is invoked.
type RepoLock struct {
Unlock func() error
Renew func(context.Context) error
}
// WriteLockManager serializes write access to a repository by acquiring
// a per-repository lock at the reference transaction preparing phase.
type WriteLockManager interface {
// Lock acquires the reference write lock in a blocking way
Lock(ctx context.Context, virtualStorage, relativePath string, txnID uint64) (RepoLock, error)
}
// NoopWriteLockManager is a no-op WriteLockManager. It always grants the
// lock immediately and is used when write serialization is not needed.
type NoopWriteLockManager struct{}
// Lock always succeeds and returns no-op functions.
func (r *NoopWriteLockManager) Lock(_ context.Context, _ string, _ string, _ uint64,
) (RepoLock, error) {
noopFn := func() error { return nil }
noopFnWithCtx := func(ctx context.Context) error { return nil }
return RepoLock{Unlock: noopFn, Renew: noopFnWithCtx}, nil
}
// RepoReferenceWriteLockManager manages per-repository reference write locks backed by
// PostgreSQL. It serializes write requests to the same repository by ensuring only one
// transaction holds the lock at a time. Callers use Lock to acquire the lock.
//
// PostgreSQL-backed locks are used instead of in-process mutexes to coordinate across
// multiple Praefect instances sharing the same database.
type RepoReferenceWriteLockManager struct {
qc glsql.Querier
handler *lockReleaseDispatcher
renewInterval time.Duration
logger log.Logger
// lockReleasingListener listens for PostgreSQL NOTIFY events when locks are released.
// Held here to expose its reconnect metrics via Collect.
lockReleasingListener *ResilientListener
// lockAcquiredAt tracks when each lock was successfully acquired, keyed by lockID
// (virtualStorage|relativePath). Used to compute hold duration at Unlock time.
lockAcquiredAt sync.Map
// lockAcquiredTotal counts tryLock attempts by result: "new_acquisition", "contended", or "error".
lockAcquiredTotal *prometheus.CounterVec
// locksHeld is the current number of locks held, per virtual storage.
locksHeld *prometheus.GaugeVec
// lockHoldDuration observes how long each lock was held (tryLock success → Unlock).
lockHoldDuration *prometheus.HistogramVec
// operationDuration observes database round-trip time per operation (trylock/unlock/renew).
operationDuration *prometheus.HistogramVec
}
// lockReleaseDispatcher implements glsql.ListenHandler and fans out PostgreSQL lock release
// notifications to goroutines waiting on a specific lock. Waiters register via
// RegisterForLockRelease and are signalled by closing their channel when the corresponding
// lock_id is deleted from the database.
type lockReleaseDispatcher struct {
mu sync.Mutex
waiters map[LockID][]chan struct{} // lockID → list of waiters
// ready is a signal channel that indicates the listener is connected.
// It is used in tests to provide proper synchronization.
ready chan struct{}
readyOnce sync.Once
}
// Notified signals all goroutines waiting on the released lock_id by closing their channels,
// then removes them from the waiters map.
func (d *lockReleaseDispatcher) Notified(n glsql.Notification) {
lockID := LockID(n.Payload)
d.mu.Lock()
chs := d.waiters[lockID]
delete(d.waiters, lockID)
d.mu.Unlock()
for _, ch := range chs {
close(ch) // wake all waiters
}
}
// RegisterForLockRelease registers the caller as a waiter for the lock identified by lockID.
// It returns a channel that will be closed when the lock is released, and a deregister
// function that the caller must invoke if it stops waiting before the channel is closed
// (e.g. on context cancellation), to remove itself from the waiters map.
func (d *lockReleaseDispatcher) RegisterForLockRelease(lock LockID) (<-chan struct{}, func()) {
ch := make(chan struct{})
d.mu.Lock()
d.waiters[lock] = append(d.waiters[lock], ch)
d.mu.Unlock()
return ch, func() {
d.mu.Lock()
defer d.mu.Unlock()
chs := d.waiters[lock]
for i, w := range chs {
if w == ch {
d.waiters[lock] = append(chs[:i], chs[i+1:]...)
if len(d.waiters[lock]) == 0 {
delete(d.waiters, lock)
}
break
}
}
}
}
// Disconnected is a no-op; waiters are woken on reconnect in Connected.
func (d *lockReleaseDispatcher) Disconnected(error) {
// Disconnected is a no-op. Waiters are woken up in Connected() once the
// listener reconnects, at which point they retry and re-register if needed.
}
// Connected would be triggered once a connection to remote service is established.
func (d *lockReleaseDispatcher) Connected() {
d.readyOnce.Do(func() { close(d.ready) }) // synchronization on tests, not used in production
// Wake all current waiters so they retry lock and discover whether the lock
// is free. This handles the case where a lock was released and its NOTIFY was
// fired while the listener was disconnected and missed. Since the listener is
// now re-subscribed, any subsequent release will be captured, so waiters that
// re-register after this point are safe.
d.mu.Lock()
waiters := d.waiters
d.waiters = make(map[LockID][]chan struct{})
d.mu.Unlock()
for _, chs := range waiters {
for _, ch := range chs {
close(ch)
}
}
}
// NewRepoReferenceWriteLockManager creates a new RepoReferenceWriteLockManager. It starts
// a background listener for lock release notifications and a background job to
// clean up expired locks. Both run until ctx is cancelled.
func NewRepoReferenceWriteLockManager(ctx context.Context, qc glsql.Querier, conf config.DB, logger log.Logger) *RepoReferenceWriteLockManager {
resilientListenerTicker := helper.NewTimerTicker(5 * time.Second)
lockReleasingListener := NewResilientListener(conf, resilientListenerTicker, logger, lockReleasingListenerReconnectsTotal)
handler := &lockReleaseDispatcher{
waiters: make(map[LockID][]chan struct{}),
ready: make(chan struct{}),
}
// In production, we start the listener asynchronously to speed up startup.
// Conceptually, we could block in the constructor until the connection is established,
// since the manager cannot function correctly without the listener. However, this has
// a downside: if the database is temporarily slow during startup, the constructor may hang.
go func() {
err := lockReleasingListener.Listen(ctx, handler, RepositoryReferenceWriteLockReleasesChannel)
if err != nil && !errors.Is(err, context.Canceled) {
logger.WithError(err).Error("notifications listener terminated")
}
}()
// Start a scheduled cleanup background job to remove expired locks.
lockRenewInterval := 20 * time.Second
cleanUpJobTicker := helper.NewTimerTicker(2 * lockRenewInterval)
go func() {
defer cleanUpJobTicker.Stop()
for {
select {
case <-ctx.Done():
return
case <-cleanUpJobTicker.C():
cleanUpExpiredRepoRefWriteLocks(ctx, qc, logger)
}
}
}()
return &RepoReferenceWriteLockManager{
qc: qc,
handler: handler,
renewInterval: lockRenewInterval,
logger: logger,
lockReleasingListener: lockReleasingListener,
lockAcquiredTotal: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gitaly_praefect_repo_reference_write_lock_acquired_total",
Help: "Total number of repository reference write locks acquired for the first time, partitioned by virtual storage.",
},
[]string{"virtual_storage", "result"},
),
locksHeld: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gitaly_praefect_repo_reference_write_locks_held",
Help: "Current number of repository reference write locks held, per virtual storage.",
},
[]string{"virtual_storage"},
),
lockHoldDuration: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gitaly_praefect_repo_reference_write_lock_hold_duration_seconds",
Help: "Duration in seconds that repository reference write locks are held (from successful acquisition to unlock).",
Buckets: prometheus.DefBuckets,
},
[]string{"virtual_storage"},
),
operationDuration: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gitaly_praefect_repo_reference_write_lock_operation_duration_seconds",
Help: "Duration in seconds of repository reference write lock database operations, by operation (trylock, unlock, renew).",
Buckets: prometheus.DefBuckets,
},
[]string{"virtual_storage", "operation"},
),
}
}
// Describe implements prometheus.Collector.
func (r *RepoReferenceWriteLockManager) Describe(descs chan<- *prometheus.Desc) {
prometheus.DescribeByCollect(r, descs)
}
// Collect implements prometheus.Collector.
func (r *RepoReferenceWriteLockManager) Collect(ch chan<- prometheus.Metric) {
r.lockAcquiredTotal.Collect(ch)
r.locksHeld.Collect(ch)
r.lockHoldDuration.Collect(ch)
r.operationDuration.Collect(ch)
r.lockReleasingListener.Collect(ch)
}
// Lock acquires the write lock for a given virtualStorage, relativePath and txnID,
// blocking until the lock is acquired or ctx is cancelled.
func (r *RepoReferenceWriteLockManager) Lock(ctx context.Context, virtualStorage, relativePath string, txnID uint64,
) (RepoLock, error) {
for {
res := r.tryLock(ctx, virtualStorage, relativePath, txnID)
if res.Err != nil {
return RepoLock{}, res.Err
}
if res.Acquired {
return RepoLock{Unlock: res.Unlock, Renew: res.Renew}, nil
}
select {
case <-ctx.Done():
res.Deregister()
return RepoLock{}, fmt.Errorf("wait for lock: %w", ctx.Err())
case <-res.NotificationCh:
// wait for the lock to release
}
}
}
type tryLockResult struct {
Acquired bool
Unlock func() error
Renew func(context.Context) error
NotificationCh <-chan struct{}
Deregister func()
Err error
}
// tryLock attempts a single non-blocking acquisition of the write lock for
// a given virtualStorage, relativePath and txnID.
// If the same txnID already holds the lock, the call is a no-op and returns
// as acquired.
//
// If tryLockResult.Err is non-nil, the request has failed and the caller should retry
// after a brief delay.
//
// If tryLockResult.Acquired is true, the lock has been successfully acquired. The caller
// should invoke either the tryLockResult.Unlock() or tryLockResult.Renew() closure.
//
// If tryLockResult.Acquired is false, the lock is held by a different txnID. The caller
// should listen on tryLockResult.NotificationCh to be notified when the lock is released.
// The caller must call tryLockResult.Deregister() if aborting the current operation: for
// example, if the request context is cancelled.
func (r *RepoReferenceWriteLockManager) tryLock(ctx context.Context, virtualStorage, relativePath string, txnID uint64,
) tryLockResult {
lockID := repoLockID(virtualStorage, relativePath)
// Register for a lock release notification before attempting the INSERT, so
// that no release event can be missed between a failed attempt and the caller
// beginning to wait, thus eliminating the race window between contention detection
// and notification.
notificationCh, deregister := r.handler.RegisterForLockRelease(lockID)
query := `
INSERT INTO repository_reference_write_locks as locks (lock_id, holder_txn_id, expired_at)
VALUES ($1, $2, NOW() + $3::interval)
ON CONFLICT (lock_id) DO UPDATE
SET holder_txn_id = EXCLUDED.holder_txn_id,
expired_at = EXCLUDED.expired_at
WHERE locks.expired_at < NOW() OR locks.holder_txn_id = $2
RETURNING lock_id, holder_txn_id, expired_at;`
start := time.Now()
rows, err := r.qc.QueryContext(ctx, query, lockID, txnID, r.renewInterval)
r.operationDuration.WithLabelValues(virtualStorage, "trylock").Observe(time.Since(start).Seconds())
if err != nil {
r.lockAcquiredTotal.WithLabelValues(virtualStorage, "error").Inc()
deregister()
return tryLockResult{
Acquired: false,
Err: fmt.Errorf("acquire repo reference write lock: %s, %w", lockID, err),
}
}
defer func() {
if err := rows.Close(); err != nil {
r.logger.WithError(err).Error("close rows")
}
}()
if rows.Next() {
_, alreadyHeld := r.lockAcquiredAt.LoadOrStore(lockID, time.Now())
if !alreadyHeld {
r.locksHeld.WithLabelValues(virtualStorage).Inc()
r.lockAcquiredTotal.WithLabelValues(virtualStorage, "new_acquisition").Inc()
}
deregister()
unlockFn := func() error {
return r.unlock(virtualStorage, relativePath, txnID)
}
renewFn := func(renewCtx context.Context) error {
return r.renew(renewCtx, virtualStorage, relativePath, txnID)
}
return tryLockResult{
Acquired: true,
Unlock: unlockFn,
Renew: renewFn,
}
}
if err := rows.Err(); err != nil {
deregister()
return tryLockResult{
Acquired: false,
Err: fmt.Errorf("acquire repo reference write lock: %s, %w", lockID, err),
}
}
r.lockAcquiredTotal.WithLabelValues(virtualStorage, "contended").Inc()
return tryLockResult{
Acquired: false,
NotificationCh: notificationCh,
Deregister: deregister,
}
}
// unlock releases the write lock held by txnID. It is a no-op if the lock is not currently held by txnID
// (e.g. it expired and was taken over by another transaction).
func (r *RepoReferenceWriteLockManager) unlock(virtualStorage string, relativePath string, txnID uint64) error {
// unlock has its own context, because the lock must be released regardless of
// the request's lifecycle. Don't use the caller's ctx because if the caller's request
// context is cancelled (e.g. client disconnected, deadline exceeded), the request is
// done but the lock row still exists in the database.
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
query := `
DELETE FROM repository_reference_write_locks
WHERE lock_id = $1 AND holder_txn_id = $2;`
lockID := repoLockID(virtualStorage, relativePath)
start := time.Now()
_, err := r.qc.ExecContext(ctx, query, lockID, txnID)
r.operationDuration.WithLabelValues(virtualStorage, "unlock").Observe(time.Since(start).Seconds())
if err != nil {
return fmt.Errorf("release repo reference write lock: %s, %w", lockID, err)
}
if acquiredAt, ok := r.lockAcquiredAt.LoadAndDelete(lockID); ok {
r.locksHeld.WithLabelValues(virtualStorage).Dec()
r.lockHoldDuration.WithLabelValues(virtualStorage).Observe(time.Since(acquiredAt.(time.Time)).Seconds())
}
return nil
}
// renew extends the expiry of the write lock held by txnID by another renewInterval.
// It returns an error if the lock is not currently held by txnID.
func (r *RepoReferenceWriteLockManager) renew(ctx context.Context, virtualStorage string, relativePath string, txnID uint64) error {
query := `
UPDATE repository_reference_write_locks as locks
SET expired_at = (NOW() + $3::interval)
WHERE locks.lock_id = $1 AND holder_txn_id = $2
RETURNING expired_at;
`
lockID := repoLockID(virtualStorage, relativePath)
start := time.Now()
rows, err := r.qc.QueryContext(ctx, query, lockID, txnID, r.renewInterval)
r.operationDuration.WithLabelValues(virtualStorage, "renew").Observe(time.Since(start).Seconds())
if err != nil {
return fmt.Errorf("renew repo reference write lock (executing query): %s, %w", lockID, err)
}
defer func() {
if err := rows.Close(); err != nil {
r.logger.WithError(err).Error("close rows")
}
}()
if rows.Next() {
return nil
}
if err := rows.Err(); err != nil {
return fmt.Errorf("renew repo reference write lock (iterating rows): %s, %w", lockID, err)
}
return fmt.Errorf("renew repo reference write lock (no rows): %s", lockID)
}
func cleanUpExpiredRepoRefWriteLocks(ctx context.Context, qc glsql.Querier, logger log.Logger) {
query := `
DELETE FROM repository_reference_write_locks
WHERE expired_at < NOW();`
result, err := qc.ExecContext(ctx, query)
if err != nil {
logger.WithError(err).Error("cleanup expired lock")
return
}
n, err := result.RowsAffected()
if err != nil {
logger.WithError(err).Error("cleanup expired lock")
return
}
logger.WithField("count", n).Info("clean up expired repository reference write locks")
}
// repoLockID constructs the lock ID used in the write lock table.
func repoLockID(virtualStorage, relativePath string) LockID {
return LockID(virtualStorage + "|" + relativePath)
}