Git designed for AI agents.
package bundleuri
import (
"context"
"errors"
"fmt"
"io"
"path/filepath"
"strings"
"github.com/feder-cr/git-ai/internal/backup"
"github.com/feder-cr/git-ai/internal/featureflag"
"github.com/feder-cr/git-ai/internal/git/gitcmd"
"github.com/feder-cr/git-ai/internal/git/localrepo"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/log"
"github.com/feder-cr/git-ai/internal/structerr"
"github.com/feder-cr/git-ai/proto/go/gitalypb"
"github.com/prometheus/client_golang/prometheus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
// GenerationManager manages bundle generation. It handles requests to
// generate bundles for a repository, and enforces concurrency by limiting one
// bundle generation per repo at any given time as well as a global limit across
// all repositories.
type GenerationManager struct {
ctx context.Context
sink *Sink
logger log.Logger
strategy GenerationStrategy
nodeManager storage.Node
bundleGenerationLatency prometheus.Histogram
bundleGenerationBytes prometheus.Counter
}
// NewGenerationManager creates a new GenerationManager
// ctx must be a cancellable context. It will be passed to the function
// generating bundles and writing bundles to storage. If ctx gets cancelled,
// all bundles currently being generated at that moment will also be cancelled.
func NewGenerationManager(ctx context.Context, sink *Sink, logger log.Logger, nodeManager storage.Node, strategy GenerationStrategy) (*GenerationManager, error) {
if sink == nil {
return nil, structerr.NewInvalidArgument("cannot create bundle manager: missing sink")
}
return &GenerationManager{
// we keep a reference to the context passed in the constructor
// because we want to be able to abort bundle generation if this
// context gets cancelled.
ctx: ctx,
sink: sink,
strategy: strategy,
logger: logger,
nodeManager: nodeManager,
bundleGenerationBytes: prometheus.NewCounter(
prometheus.CounterOpts{
Name: "gitaly_bundle_generation_bytes_total",
Help: "Total number of bytes written to cloud storage",
},
),
bundleGenerationLatency: prometheus.NewHistogram(
prometheus.HistogramOpts{
Name: "gitaly_bundle_generation_seconds",
Buckets: []float64{1, 30, 60, 5 * 60, 30 * 60, 60 * 60, 2 * 3600, 5 * 3600, 12 * 3600, 24 * 3600},
},
),
}, nil
}
// Describe is used to describe Prometheus metrics.
func (g *GenerationManager) Describe(descs chan<- *prometheus.Desc) {
prometheus.DescribeByCollect(g, descs)
}
// Collect is used to collect Prometheus metrics.
func (g *GenerationManager) Collect(metrics chan<- prometheus.Metric) {
g.bundleGenerationLatency.Collect(metrics)
g.bundleGenerationBytes.Collect(metrics)
}
// GenerateWithStrategy runs the strategy within the manager to determine if a bundle must
// be generated.
func (g *GenerationManager) GenerateWithStrategy(ctx context.Context, repo *localrepo.Repo) error {
if featureflag.BundleGeneration.IsEnabled(ctx) {
return g.strategy.Evaluate(ctx, repo, g.Generate)
}
return nil
}
// Generate will generate a bundle for the given `repo`. This method does not attempt to
// verify any feature flag or conditions. Calling this method WILL generate a bundle.
func (g *GenerationManager) Generate(ctx context.Context, repo *localrepo.Repo) (returnErr error) {
ref, err := repo.HeadReference(ctx)
if err != nil {
return fmt.Errorf("resolve HEAD ref: %w", err)
}
repoProto, ok := repo.Repository.(*gitalypb.Repository)
if !ok {
return fmt.Errorf("unexpected repository type %t", repo.Repository)
}
// We need a distinct context from the request's context (ctx).
// This is because if we use the request's context during bundle generation,
// and if this request runs inside a transaction, the snapshot that holds a
// copy of the repo will be deleted at the end of the transaction, but the bundle
// might not have finished generating yet. So we need a new context, and a new
// transaction inside that context, so we can have a snapshot that holds for
// the duration of the bundle generation. We also want this new context to be
// `from` the manager's context to inherit its cancellation.
gCtx, cancel := context.WithCancel(g.ctx)
defer cancel()
// We must use `ctx` here and not `gCtx`, because `ctx` is the context
// of the gRPC request, and this is what we want.
bundlePath := bundleRelativePath(ctx, repo, defaultBundle)
if tx := storage.ExtractTransaction(ctx); tx != nil {
if g.nodeManager == nil {
g.logger.WithError(err).Error("generate bundle: nil node manager within transaction")
return nil
}
originalRepo := tx.OriginalRepository(repoProto)
strg, err := g.nodeManager.GetStorage(originalRepo.GetStorageName())
if err != nil {
g.logger.WithError(err).Error("generate bundle: error getting storage")
return nil
}
// Create the transaction on the new context created above
ntx, err := strg.Begin(gCtx, storage.TransactionOptions{
ReadOnly: true,
RelativePath: originalRepo.GetRelativePath(),
})
if err != nil {
g.logger.WithError(err).Error("generate bundle: no transaction found")
return nil
}
// We only create a new transaction to have a dedicated snapshot during
// bundle generation. So once the bundle is generated, we must abort
// to free the snapshot.
defer func() { _ = ntx.Rollback(gCtx) }()
// We must use `ctx` here and not `gCtx`, because `ctx` is the context
// of the gRPC request, and this is what we want.
bundlePath = bundleRelativePath(ctx, originalRepo, defaultBundle)
}
writer := backup.NewLazyWriter(func() (io.WriteCloser, error) {
return g.sink.getWriter(gCtx, bundlePath)
})
defer func() {
if err := writer.Close(); err != nil && returnErr == nil {
returnErr = fmt.Errorf("write bundle: %w", err)
}
}()
opts := localrepo.CreateBundleOpts{
Patterns: strings.NewReader(ref.String()),
}
timer := prometheus.NewTimer(g.bundleGenerationLatency)
err = repo.CreateBundle(gCtx, writer, &opts)
switch {
case errors.Is(err, localrepo.ErrEmptyBundle):
return structerr.NewFailedPrecondition("ref %q does not exist: %w", ref, err)
case err != nil:
g.logger.WithField("gl_project_path", repo.GetGlProjectPath()).
WithError(err).
Error("failed to generate bundle")
return structerr.NewInternal("%w", err)
}
timer.ObserveDuration()
g.bundleGenerationBytes.Add(float64(writer.BytesWritten()))
g.logger.WithField("gl_project_path", repo.GetGlProjectPath()).
Info("bundle generated successfully")
return nil
}
// SignedURL returns a public URL to give anyone access to download the bundle from.
func (g *GenerationManager) SignedURL(ctx context.Context, repo storage.Repository) (string, error) {
relativePath := bundleRelativePath(ctx, repo, defaultBundle)
repoProto, ok := repo.(*gitalypb.Repository)
if !ok {
return "", fmt.Errorf("unexpected repository type %t", repo)
}
if tx := storage.ExtractTransaction(ctx); tx != nil {
origRepo := tx.OriginalRepository(repoProto)
relativePath = bundleRelativePath(ctx, origRepo, defaultBundle)
}
return g.sink.signedURL(ctx, relativePath)
}
// UploadPackGitConfig is a helper function to provide all required, and computed, configurations
// to inject into the `git-upload-pack` command in order to advertise `bundle-uri` and provide
// the URI for the bundle for the given repository.
func (g *GenerationManager) UploadPackGitConfig(ctx context.Context, repo storage.Repository) []gitcmd.ConfigPair {
uri, err := g.SignedURL(ctx, repo)
if err != nil {
if st, ok := status.FromError(err); !ok || st.Code() != codes.NotFound {
g.logger.WithField("bundle_uri_error", err)
}
return CapabilitiesGitConfig(ctx, false)
}
return UploadPackGitConfig(ctx, uri)
}
// bundleRelativePath returns a relative path of the bundle-URI bundle inside the bucket.
func bundleRelativePath(ctx context.Context, repo storage.Repository, name string) string {
repoPath := filepath.Join(storageNameForBundle(ctx, repo), repo.GetRelativePath())
return filepath.Join(repoPath, "uri", name+".bundle")
}
// storageNameForBundle returns the name of the storage of the repository
// to be used to generate the bundle path. If the context contains gRPC
// metadata in which the Praefect virtual storage is present, it uses that storage
// name. Else it defaults to the repository's storage name.
// Using the Praefect virtual storage name is an optimization technique to
// avoid generating one bundle per Gitaly node when behind Praefect.
func storageNameForBundle(ctx context.Context, repo storage.Repository) string {
storageName := repo.GetStorageName()
md, _ := metadata.FromIncomingContext(ctx)
vs := md.Get(VirtualStorageKey)
if len(vs) > 0 {
storageName = vs[0]
}
return storageName
}