Skip to content

Git designed for AI agents.

10.8 KB
Raw Blame
// Package eager runs the work that Graft does right after references are
// committed. Every committed reference update of a repository reaches the
// Observer from the reference-transaction hook. The Observer enqueues derived
// data for the new commits that the updates made reachable and schedules an
// incremental commit-graph write for the repository. Lands go through the same
// hook, so they trigger both without a separate path. The commit-graph write is
// debounced per repository, so a burst of lands writes it once.
//
// Nothing here ever fails or blocks the reference update: the hook only
// schedules the work, and errors are logged.
package eager

import (
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/feder-cr/git-ai/internal/git"
	"github.com/feder-cr/git-ai/internal/gitaly/storage"
	"github.com/feder-cr/git-ai/internal/graft/changelog"
	"github.com/feder-cr/git-ai/internal/log"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	// DefaultDebounce is the quiet period after the last committed update of a
	// repository before its commit-graph is written. Writes of one repository
	// are therefore at least this far apart.
	DefaultDebounce = 5 * time.Second
	// DefaultMaxWait bounds how long a commit-graph write can be postponed by
	// a steady stream of updates.
	DefaultMaxWait = 30 * time.Second
	// MaxNewCommitsPerUpdate bounds the commits listed for one reference update.
	MaxNewCommitsPerUpdate = 1000
	// DefaultListWorkers is the number of listings of new commits that run at
	// the same time.
	DefaultListWorkers = 2
	// DefaultListQueueSize is the number of committed transactions whose new
	// commits wait for a listing worker.
	DefaultListQueueSize = 1000

	// listTimeout bounds the listing of the new commits of one update.
	listTimeout = time.Minute
)

var droppedListingsDesc = prometheus.NewDesc("graft_eager_dropped_listings_total",
	"Number of committed reference transactions whose new commits were not listed for the eager "+
		"derived-data warmers because the listing queue was full.", nil, nil)

// Enqueuer schedules derived data for commits of a repository. The derived
// Pool implements it. Enqueue never blocks.
type Enqueuer interface {
	Enqueue(repo storage.Repository, commits []git.ObjectID)
}

// NewCommitsFunc lists the commits that moving the reference from oldOID to
// newOID made reachable: those of newOID that oldOID does not reach, at most
// MaxNewCommitsPerUpdate of them. A zero oldOID lists the history of newOID.
type NewCommitsFunc func(ctx context.Context, repo storage.Repository, oldOID, newOID string) ([]git.ObjectID, error)

// CommitGraphWriter writes the commit-graph of the repository.
type CommitGraphWriter func(ctx context.Context, repo storage.Repository) error

// Options configures an Observer.
type Options struct {
	// Eager enables enqueueing of derived data for the new commits of the
	// updated references, see [graft.derived] eager.
	Eager bool
	// CommitGraph enables the debounced commit-graph write, see
	// [graft.derived] commit_graph. Off by default: a zero Options writes no
	// commit-graph outside housekeeping.
	CommitGraph bool
	// Debounce is the quiet period before a commit-graph write. Zero means
	// DefaultDebounce.
	Debounce time.Duration
	// MaxWait bounds the postponement of a commit-graph write. Zero means
	// DefaultMaxWait.
	MaxWait time.Duration
	// ListWorkers is the number of listings of new commits, each a git
	// rev-list, that run at the same time. Zero means DefaultListWorkers.
	ListWorkers int
	// ListQueueSize is the number of committed transactions that wait for a
	// listing worker. When the queue is full, the transaction is dropped and
	// counted. Zero means DefaultListQueueSize.
	ListQueueSize int
}

// Deps are the collaborators of an Observer. A nil collaborator disables the
// trigger that needs it.
type Deps struct {
	Enqueuer         Enqueuer
	ListNewCommits   NewCommitsFunc
	WriteCommitGraph CommitGraphWriter
}

// Observer receives the committed reference updates of repositories.
//
// The new commits are listed by a fixed set of workers from a bounded queue, so
// a burst of committed transactions never runs more than ListWorkers git
// processes at a time. A transaction that finds the queue full is dropped and
// counted: its commits are computed lazily by the next reader that needs them.
//
// Observer implements prometheus.Collector with graft_eager_dropped_listings_total.
type Observer struct {
	logger         log.Logger
	eager          bool
	enqueuer       Enqueuer
	listNewCommits NewCommitsFunc
	scheduler      *Scheduler

	// listQueue holds the transactions whose new commits wait for a listing
	// worker.
	listQueue chan listJob
	// droppedListings counts the transactions dropped because listQueue was full.
	droppedListings atomic.Int64

	// ctx is cancelled by Close. It is the parent of the background listings.
	ctx    context.Context
	cancel context.CancelFunc

	// mu guards closed. wg tracks the listing workers.
	mu     sync.Mutex
	closed bool
	wg     sync.WaitGroup
}

// listJob is the listing of the new commits of one committed transaction.
type listJob struct {
	repo    storage.Repository
	targets []updateTarget
}

// NewObserver returns an Observer. The Observer owns its scheduler and its
// listing workers, so the caller must Close it.
func NewObserver(ctx context.Context, logger log.Logger, opts Options, deps Deps) *Observer {
	ctx, cancel := context.WithCancel(ctx)

	o := &Observer{
		logger:         logger,
		eager:          opts.Eager && deps.Enqueuer != nil && deps.ListNewCommits != nil,
		enqueuer:       deps.Enqueuer,
		listNewCommits: deps.ListNewCommits,
		ctx:            ctx,
		cancel:         cancel,
	}

	if o.eager {
		workers := opts.ListWorkers
		if workers <= 0 {
			workers = DefaultListWorkers
		}
		queueSize := opts.ListQueueSize
		if queueSize <= 0 {
			queueSize = DefaultListQueueSize
		}

		o.listQueue = make(chan listJob, queueSize)
		for i := 0; i < workers; i++ {
			o.wg.Add(1)
			go o.listWorker()
		}
	}

	if opts.CommitGraph && deps.WriteCommitGraph != nil {
		o.scheduler = NewScheduler(ctx, logger, deps.WriteCommitGraph, opts.Debounce, opts.MaxWait)
	}

	return o
}

// ObserveCommitted is called with the reference updates that were committed to
// repo. It never blocks on Git and never fails the update: the new commits are
// listed and enqueued in the background, and a panic is recovered and logged.
// Deleted references do not enqueue derived data, but they still schedule a
// commit-graph write because they change reachability.
func (o *Observer) ObserveCommitted(ctx context.Context, repo storage.Repository, updates []changelog.Update) {
	if o == nil || len(updates) == 0 {
		return
	}

	defer func() {
		if r := recover(); r != nil {
			o.logger.WithField("panic", fmt.Sprint(r)).ErrorContext(ctx, "eager work after committed updates panicked")
		}
	}()

	if o.eager {
		o.enqueueNewCommits(ctx, repo, updates)
	}

	if o.scheduler != nil {
		o.scheduler.Schedule(repo)
	}
}

// Close stops the background work. Listings in progress are cancelled, queued
// listings and pending commit-graph writes are dropped, since derived data is
// computed lazily and the commit-graph is only an index that Git rebuilds
// lazily, and Close waits for the work in progress. Close is idempotent.
func (o *Observer) Close() {
	if o == nil {
		return
	}

	o.mu.Lock()
	if o.closed {
		o.mu.Unlock()
		return
	}
	o.closed = true
	o.mu.Unlock()

	o.cancel()
	o.wg.Wait()

	if o.scheduler != nil {
		o.scheduler.Close()
	}
}

// enqueueNewCommits queues the listing of the new commits of the updated
// references for the listing workers. It never blocks: when the queue is full,
// the transaction is dropped and counted.
func (o *Observer) enqueueNewCommits(ctx context.Context, repo storage.Repository, updates []changelog.Update) {
	targets := updatedTargets(updates)
	if len(targets) == 0 {
		return
	}

	o.mu.Lock()
	closed := o.closed
	o.mu.Unlock()
	if closed {
		return
	}

	select {
	case o.listQueue <- listJob{repo: repo, targets: targets}:
	default:
		o.droppedListings.Add(1)
		o.logger.WithField("storage", repo.GetStorageName()).WithField("relative_path", repo.GetRelativePath()).
			DebugContext(ctx, "eager: listing queue full, new commits are not warmed")
	}
}

// listWorker lists and enqueues the new commits of queued transactions until
// Close.
func (o *Observer) listWorker() {
	defer o.wg.Done()

	for {
		select {
		case <-o.ctx.Done():
			return
		case job := <-o.listQueue:
			if o.ctx.Err() != nil {
				return
			}
			o.runListJob(job)
		}
	}
}

// runListJob runs one listing. A panic is recovered and logged, so that the
// worker keeps running.
func (o *Observer) runListJob(job listJob) {
	defer func() {
		if r := recover(); r != nil {
			o.logger.WithField("panic", fmt.Sprint(r)).ErrorContext(o.ctx, "eager: enqueueing new commits panicked")
		}
	}()

	o.enqueueTargets(o.ctx, job.repo, job.targets)
}

// Describe implements prometheus.Collector.
func (o *Observer) Describe(ch chan<- *prometheus.Desc) {
	ch <- droppedListingsDesc
}

// Collect implements prometheus.Collector.
func (o *Observer) Collect(ch chan<- prometheus.Metric) {
	ch <- prometheus.MustNewConstMetric(droppedListingsDesc, prometheus.CounterValue, float64(o.droppedListings.Load()))
}

// enqueueTargets lists the new commits of every target and enqueues the distinct
// ones together.
func (o *Observer) enqueueTargets(ctx context.Context, repo storage.Repository, targets []updateTarget) {
	listCtx, cancel := context.WithTimeout(o.ctx, listTimeout)
	defer cancel()

	var commits []git.ObjectID
	seen := make(map[git.ObjectID]struct{})

	for _, target := range targets {
		listed, err := o.listNewCommits(listCtx, repo, target.oldOID, target.newOID)
		if err != nil {
			o.logger.WithError(err).WithField("reference", target.reference).WarnContext(ctx, "eager: listing new commits failed")
			continue
		}

		for _, oid := range listed {
			if _, ok := seen[oid]; ok {
				continue
			}
			seen[oid] = struct{}{}
			commits = append(commits, oid)
		}
	}

	if len(commits) == 0 || listCtx.Err() != nil {
		return
	}

	o.enqueuer.Enqueue(repo, commits)
}

// updateTarget is one committed update of a reference to a new value.
type updateTarget struct {
	reference string
	oldOID    string
	newOID    string
}

// updatedTargets returns the updates that set a reference to a new value, in
// the order of the updates. Deletions are skipped, and so are the updates that
// repeat an earlier one.
func updatedTargets(updates []changelog.Update) []updateTarget {
	var targets []updateTarget
	seen := make(map[updateTarget]struct{}, len(updates))

	for _, update := range updates {
		if changelog.IsZeroOID(update.NewOID) {
			continue
		}

		target := updateTarget{reference: update.Reference, oldOID: update.OldOID, newOID: update.NewOID}
		if _, ok := seen[target]; ok {
			continue
		}

		seen[target] = struct{}{}
		targets = append(targets, target)
	}

	return targets
}