Git designed for AI agents.
package backup
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/log"
"github.com/feder-cr/git-ai/proto/go/gitalypb"
)
// repositoryKey uniquely identifies a repository, and is used here to key the
// map of processed repos.
type repositoryKey string
// Strategy used to create/restore backups
type Strategy interface {
Create(context.Context, *CreateRequest) error
Restore(context.Context, *RestoreRequest) error
}
// IDHandler handles backup-run-level operations scoped to an entire
// backup run rather than individual repositories.
type IDHandler interface {
WriteBackupID(ctx context.Context, req *WriteBackupIDRequest) error
ReadLatestBackupID(ctx context.Context, pathPrefix string) (string, error)
}
// Operator is the full interface implemented by all backup managers. CLI code
// uses this so a single variable covers both per-repository and run-level
// operations.
type Operator interface {
Strategy
IDHandler
}
// CreateRequest is the request to create a backup
type CreateRequest struct {
// Server contains gitaly server connection information required to call
// RPCs in the non-local backup.Manager configuration.
Server storage.ServerInfo
// Repository is the repository to be backed up.
Repository *gitalypb.Repository
// VanityRepository is used to determine the backup path.
VanityRepository *gitalypb.Repository
// Incremental when true will create an increment on the specified full backup.
Incremental bool
// BackupID is used to determine a unique path for the backup when a full
// backup is created.
BackupID string
// LatestBackupID is used to determine the last backup if incremental is true.
// If incremental is true, LastBackupID is empty and the fallback FindLatest
// also returns ErrDoesntExist, it would fallback to creating full backup.
LatestBackupID string
// PathPrefix is used to determine the destination directory for the backup
PathPrefix string
}
// WriteBackupIDRequest is the request to write the backup ID marker of a
// completed backup run.
type WriteBackupIDRequest struct {
// Server contains gitaly server connection information required to call
// RPCs in the server-side configuration. When zero, the server of
// StorageName is looked up in the context.
Server storage.ServerInfo
// StorageName is the storage the RPC is routed to in the server-side
// configuration. It must be a storage that the backup run backed up a
// repository to.
StorageName string
// BackupID is the ID of the completed backup run.
BackupID string
// PathPrefix is used to determine the destination directory for the marker.
PathPrefix string
}
// RestoreRequest is the request to restore from a backup
type RestoreRequest struct {
// Server contains gitaly server connection information required to call
// RPCs in the non-local backup.Manager configuration.
Server storage.ServerInfo
// Repository is the repository to be restored.
Repository *gitalypb.Repository
// VanityRepository is used to determine the backup path.
VanityRepository *gitalypb.Repository
// AlwaysCreate forces the repository to be created even if no bundle for
// it exists. See https://gitlab.com/gitlab-org/gitlab/-/issues/357044
AlwaysCreate bool
// BackupID is the ID of the full backup to restore. If not specified, the
// latest backup is restored..
BackupID string
// UseLatest is a flag to distinguish if ErrDoesntExist error returned for a backupID
// provided by user, or the backupID returned from the ReadLatestBackupID function.
// When user provides a backupID, UseLatest is always false. If it is true and the Find
// call resulted in ErrDoesntExist, restore falls back to calling FindLatest for that
// specific repository.
UseLatest bool
// PathPrefix is used to determine the destination directory for the backup
PathPrefix string
// ReferenceBackend is the backend to recreate the repository with, when the restore has to
// create it. The zero value leaves the choice to the node. A repository that already
// exists keeps its own.
ReferenceBackend git.ReferenceBackend
}
// Command handles a specific backup operation
type Command interface {
Repository() *gitalypb.Repository
Name() string
Execute(context.Context) error
}
// CreateCommand creates a backup for a repository
type CreateCommand struct {
strategy Strategy
request CreateRequest
}
// NewCreateCommand builds a CreateCommand
func NewCreateCommand(strategy Strategy, request CreateRequest) *CreateCommand {
return &CreateCommand{
strategy: strategy,
request: request,
}
}
// Repository is the repository that will be acted on
func (cmd CreateCommand) Repository() *gitalypb.Repository {
return cmd.request.Repository
}
// Name is the name of the command
func (cmd CreateCommand) Name() string {
return "create"
}
// Execute performs the backup
func (cmd CreateCommand) Execute(ctx context.Context) error {
return cmd.strategy.Create(ctx, &cmd.request)
}
// RestoreCommand restores a backup for a repository
type RestoreCommand struct {
strategy Strategy
request RestoreRequest
}
// NewRestoreCommand builds a RestoreCommand
func NewRestoreCommand(strategy Strategy, request RestoreRequest) *RestoreCommand {
return &RestoreCommand{
strategy: strategy,
request: request,
}
}
// Repository is the repository that will be acted on
func (cmd RestoreCommand) Repository() *gitalypb.Repository {
return cmd.request.Repository
}
// Name is the name of the command
func (cmd RestoreCommand) Name() string {
return "restore"
}
// Execute performs the restore
func (cmd RestoreCommand) Execute(ctx context.Context) error {
return cmd.strategy.Restore(ctx, &cmd.request)
}
// commandErrors represents a summary of errors by repository
//
//nolint:errname
type commandErrors struct {
errs []error
mu sync.Mutex
}
// AddError adds an error associated with a repository to the summary.
func (c *commandErrors) AddError(repo *gitalypb.Repository, err error) {
c.mu.Lock()
defer c.mu.Unlock()
if repo.GetGlProjectPath() != "" {
err = fmt.Errorf("%s (%s): %w", repo.GetRelativePath(), repo.GetGlProjectPath(), err)
} else {
err = fmt.Errorf("%s: %w", repo.GetRelativePath(), err)
}
c.errs = append(c.errs, err)
}
func (c *commandErrors) Error() string {
var builder strings.Builder
_, _ = fmt.Fprintf(&builder, "%d failures encountered:\n", len(c.errs))
for _, err := range c.errs {
_, _ = fmt.Fprintf(&builder, " - %s\n", err.Error())
}
return builder.String()
}
type contextCommand struct {
Command Command
Context context.Context
}
// Pipeline is a pipeline for running backup and restore jobs.
type Pipeline struct {
log log.Logger
parallel int
parallelStorage int
// totalWorkers allows the total number of parallel jobs to be
// limited. This allows us to create the required workers for
// each storage, while still limiting the absolute parallelism.
totalWorkers chan struct{}
workerWg sync.WaitGroup
workersByStorage map[string]chan *contextCommand
workersByStorageMu sync.Mutex
// done signals that no more commands will be provided to the Pipeline via
// Handle(), and the pipeline should wait for workers to complete and exit.
done chan struct{}
pipelineError error
cmdErrors *commandErrors
processedRepos map[string]map[repositoryKey]struct{}
processedReposMu sync.Mutex
}
// NewPipeline creates a pipeline that executes backup and restore jobs.
// The pipeline executes sequentially by default, but can be made concurrent
// by calling WithConcurrency() after initialisation.
func NewPipeline(log log.Logger, opts ...PipelineOption) (*Pipeline, error) {
p := &Pipeline{
log: log,
// Default to no concurrency.
parallel: 1,
parallelStorage: 0,
done: make(chan struct{}),
workersByStorage: make(map[string]chan *contextCommand),
cmdErrors: &commandErrors{},
processedRepos: make(map[string]map[repositoryKey]struct{}),
}
for _, opt := range opts {
if err := opt(p); err != nil {
return nil, err
}
}
return p, nil
}
// PipelineOption represents an optional configuration parameter for the Pipeline.
type PipelineOption func(*Pipeline) error
// WithConcurrency configures the pipeline to run backup and restore jobs concurrently.
// total defines the absolute maximum number of jobs that the pipeline should execute
// concurrently. perStorage defines the number of jobs per Gitaly storage that the
// pipeline should attempt to execute concurrently.
//
// For example, in a Gitaly deployment with 2 storages, WithConcurrency(3, 2) means
// that at most 3 jobs will execute concurrently, despite 2 concurrent jobs being allowed
// per storage (2*2=4).
func WithConcurrency(total, perStorage int) PipelineOption {
return func(p *Pipeline) error {
if total == 0 && perStorage == 0 {
return errors.New("total and perStorage cannot both be 0")
}
p.parallel = total
p.parallelStorage = perStorage
if total > 0 && perStorage > 0 {
// When both values are provided, we ensure that total limits
// the global concurrency.
p.totalWorkers = make(chan struct{}, total)
}
return nil
}
}
// Handle queues a request to create a backup. Commands either processed sequentially
// or concurrently, if WithConcurrency() was called.
func (p *Pipeline) Handle(ctx context.Context, cmd Command) {
ch := p.getWorker(cmd.Repository().GetStorageName())
select {
case <-ctx.Done():
p.setErr(ctx.Err())
case ch <- &contextCommand{
Command: cmd,
Context: ctx,
}:
}
}
// Done waits for any in progress jobs to complete then reports any accumulated errors
func (p *Pipeline) Done() (processedRepos map[string]map[repositoryKey]struct{}, err error) {
close(p.done)
p.workerWg.Wait()
if p.pipelineError != nil {
return p.processedRepos, fmt.Errorf("pipeline: %w", p.pipelineError)
}
if len(p.cmdErrors.errs) > 0 {
return p.processedRepos, fmt.Errorf("pipeline: %w", p.cmdErrors)
}
return p.processedRepos, nil
}
// getWorker finds the channel associated with a storage. When no channel is
// found, one is created and n-workers are started to process requests.
// If parallelStorage is 0, a channel is created against a pseudo-storage to
// enforce the number of total concurrent jobs.
func (p *Pipeline) getWorker(storage string) chan<- *contextCommand {
p.workersByStorageMu.Lock()
defer p.workersByStorageMu.Unlock()
workers := p.parallelStorage
if p.parallelStorage == 0 {
// if the workers are not limited by storage, then pretend there is a single storage with `parallel` workers
storage = ""
workers = p.parallel
}
ch, ok := p.workersByStorage[storage]
if !ok {
ch = make(chan *contextCommand)
p.workersByStorage[storage] = ch
for i := 0; i < workers; i++ {
p.workerWg.Add(1)
go p.worker(ch)
}
}
return ch
}
func (p *Pipeline) worker(ch <-chan *contextCommand) {
defer p.workerWg.Done()
for {
select {
case <-p.done:
return
case cmd := <-ch:
p.processCommand(cmd.Context, cmd.Command)
}
}
}
func (p *Pipeline) processCommand(ctx context.Context, cmd Command) {
p.acquireWorkerSlot()
defer p.releaseWorkerSlot()
log := p.cmdLogger(cmd)
log.Info(fmt.Sprintf("started %s", cmd.Name()))
if err := cmd.Execute(ctx); err != nil {
if errors.Is(err, ErrSkipped) {
log.Warn(fmt.Sprintf("skipped %s", cmd.Name()))
} else {
log.WithError(err).Error(fmt.Sprintf("%s failed", cmd.Name()))
p.addError(cmd.Repository(), err)
}
return
}
storageName := cmd.Repository().GetStorageName()
p.processedReposMu.Lock()
if _, ok := p.processedRepos[storageName]; !ok {
p.processedRepos[storageName] = make(map[repositoryKey]struct{})
}
p.processedRepos[storageName][NewRepositoryKey(cmd.Repository())] = struct{}{}
p.processedReposMu.Unlock()
log.Info(fmt.Sprintf("completed %s", cmd.Name()))
}
func (p *Pipeline) setErr(err error) {
if p.pipelineError != nil {
return
}
p.pipelineError = err
}
func (p *Pipeline) addError(repo *gitalypb.Repository, err error) {
p.cmdErrors.AddError(repo, err)
}
func (p *Pipeline) cmdLogger(cmd Command) log.Logger {
return p.log.WithFields(log.Fields{
"command.name": cmd.Name(),
"storage_name": cmd.Repository().GetStorageName(),
"relative_path": cmd.Repository().GetRelativePath(),
"gl_project_path": cmd.Repository().GetGlProjectPath(),
})
}
// acquireWorkerSlot queues the worker until a slot is available.
func (p *Pipeline) acquireWorkerSlot() {
if p.totalWorkers == nil {
return
}
p.totalWorkers <- struct{}{}
}
// releaseWorkerSlot releases the worker slot.
func (p *Pipeline) releaseWorkerSlot() {
if p.totalWorkers == nil {
return
}
<-p.totalWorkers
}
// NewRepositoryKey returns a unique identifier for the provided repo.
func NewRepositoryKey(repo *gitalypb.Repository) repositoryKey {
return repositoryKey(repo.GetStorageName() + "-" + repo.GetRelativePath())
}