Git designed for AI agents.
package gitaly
import (
"context"
"fmt"
"os"
"os/signal"
"runtime/debug"
"time"
"github.com/feder-cr/git-ai"
gitalyclient "github.com/feder-cr/git-ai/client"
"github.com/feder-cr/git-ai/internal/backup"
"github.com/feder-cr/git-ai/internal/bootstrap"
"github.com/feder-cr/git-ai/internal/bootstrap/starter"
"github.com/feder-cr/git-ai/internal/bundleuri"
"github.com/feder-cr/git-ai/internal/burdenmonitor"
"github.com/feder-cr/git-ai/internal/cache"
"github.com/feder-cr/git-ai/internal/cgroups"
"github.com/feder-cr/git-ai/internal/git/catfile"
"github.com/feder-cr/git-ai/internal/git/gitcmd"
housekeepingmgr "github.com/feder-cr/git-ai/internal/git/housekeeping/manager"
"github.com/feder-cr/git-ai/internal/git/localrepo"
gitmvcc "github.com/feder-cr/git-ai/internal/git/mvcc"
"github.com/feder-cr/git-ai/internal/gitaly/config"
"github.com/feder-cr/git-ai/internal/gitaly/config/sentry"
"github.com/feder-cr/git-ai/internal/gitaly/hook"
"github.com/feder-cr/git-ai/internal/gitaly/hook/updateref"
"github.com/feder-cr/git-ai/internal/gitaly/maintenance"
"github.com/feder-cr/git-ai/internal/gitaly/server"
"github.com/feder-cr/git-ai/internal/gitaly/service"
"github.com/feder-cr/git-ai/internal/gitaly/service/setup"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/gitaly/storage/counter"
"github.com/feder-cr/git-ai/internal/gitaly/storage/mdfile"
"github.com/feder-cr/git-ai/internal/gitaly/storage/relational"
"github.com/feder-cr/git-ai/internal/gitaly/transaction"
"github.com/feder-cr/git-ai/internal/gitlab"
"github.com/feder-cr/git-ai/internal/graft/corestore"
"github.com/feder-cr/git-ai/internal/graft/corestore/checkpoint"
"github.com/feder-cr/git-ai/internal/graft/corestore/gate"
"github.com/feder-cr/git-ai/internal/graft/corestore/gc"
"github.com/feder-cr/git-ai/internal/graft/corestore/gitops"
"github.com/feder-cr/git-ai/internal/graft/corestore/localrefs"
storagemaintenance "github.com/feder-cr/git-ai/internal/graft/corestore/maintenance"
corestoreopen "github.com/feder-cr/git-ai/internal/graft/corestore/open"
"github.com/feder-cr/git-ai/internal/graft/corestore/reviewroots"
corestoresyncer "github.com/feder-cr/git-ai/internal/graft/corestore/syncer"
"github.com/feder-cr/git-ai/internal/graft/corestore/voter"
"github.com/feder-cr/git-ai/internal/graft/derived"
_ "github.com/feder-cr/git-ai/internal/graft/derived/graph" // registers the graph warmer
"github.com/feder-cr/git-ai/internal/graft/eager"
"github.com/feder-cr/git-ai/internal/graft/landlane"
"github.com/feder-cr/git-ai/internal/graft/policystore"
"github.com/feder-cr/git-ai/internal/graft/resolver"
"github.com/feder-cr/git-ai/internal/grpc/backchannel"
"github.com/feder-cr/git-ai/internal/grpc/client"
housekeepingmw "github.com/feder-cr/git-ai/internal/grpc/middleware/housekeeping"
"github.com/feder-cr/git-ai/internal/grpc/middleware/limithandler"
"github.com/feder-cr/git-ai/internal/grpc/middleware/mvcc"
"github.com/feder-cr/git-ai/internal/grpc/protoregistry"
"github.com/feder-cr/git-ai/internal/helper"
"github.com/feder-cr/git-ai/internal/helper/env"
"github.com/feder-cr/git-ai/internal/limiter"
"github.com/feder-cr/git-ai/internal/loadmonitor"
"github.com/feder-cr/git-ai/internal/log"
"github.com/feder-cr/git-ai/internal/streamcache"
"github.com/feder-cr/git-ai/internal/tempdir"
"github.com/feder-cr/git-ai/internal/tracing"
"github.com/feder-cr/git-ai/internal/version"
"github.com/go-enry/go-license-detector/v4/licensedb"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/urfave/cli/v3"
"gitlab.com/gitlab-org/labkit/fips"
"gitlab.com/gitlab-org/labkit/monitoring"
"go.uber.org/automaxprocs/maxprocs"
"google.golang.org/grpc"
// Import to register the proxy codec with gRPC.
_ "github.com/feder-cr/git-ai/proxy"
)
func newServeCommand() *cli.Command {
return &cli.Command{
Name: "serve",
Usage: "launch the server daemon",
UsageText: `gitaly serve <gitaly_config_file>
Example: gitaly serve gitaly.config.toml`,
Description: "Launch the Gitaly server daemon.",
Action: serveAction,
}
}
// exitWithError logs an error and returns a cli.Exit with the appropriate exit code.
// If a logger is available, it uses the logger's structured logging. Otherwise, it
// returns a plain error message.
func exitWithError(logger log.Logger, msg string, err error) error {
if logger != nil {
logger.WithError(err).Error(msg)
return cli.Exit("", 1)
}
return cli.Exit(fmt.Errorf("%s: %w", msg, err), 1)
}
func loadConfig(configPath string) (config.Cfg, error) {
cfgFile, err := os.Open(configPath)
if err != nil {
return config.Cfg{}, err
}
defer cfgFile.Close()
cfg, err := config.Load(cfgFile)
if err != nil {
return config.Cfg{}, err
}
if err := cfg.Validate(); err != nil {
return config.Cfg{}, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
func serveAction(ctx context.Context, cmd *cli.Command) error {
if cmd.NArg() != 1 || cmd.Args().First() == "" {
cli.ShowSubcommandHelpAndExit(cmd, 2)
}
// The reload signal is caught from the start: its default action ends the process, and an
// operator or gitaly-wrapper may send it while the server starts.
policySignals := notifyPolicyReload()
defer signal.Stop(policySignals)
cfg, logger, err := configure(cmd.Args().First())
if err != nil {
return exitWithError(logger, "failed to configure Gitaly", err)
}
// Flush any buffered log output to disk before the process exits. This is
// registered early so that it runs after all other deferred cleanup as well
// as the final shutdown log lines, ensuring we don't lose any logs on
// shutdown.
defer func() {
if err := logger.Flush(); err != nil {
logger.WithError(err).Warn("failed to flush logs to disk on shutdown")
}
}()
tracingCloser, err := tracing.InitializeTracerProvider(context.Background(), "gitaly")
if err != nil {
logger.WithError(err).Error("error initializing distributed tracing")
}
defer func() { _ = tracingCloser.Close() }()
if cfg.Auth.Transitioning && len(cfg.Auth.GetToken()) > 0 {
logger.Warn("Authentication is enabled but not enforced because transitioning=true. Gitaly will accept unauthenticated requests.")
}
logger.WithField("version", version.GetVersion()).Info("Starting Gitaly")
fips.Check()
if err := run(cmd, cfg, logger, policySignals); err != nil {
return exitWithError(logger, "unclean Gitaly shutdown", err)
}
logger.Info("Gitaly shutdown")
return nil
}
func configure(configPath string) (config.Cfg, log.Logger, error) {
cfg, err := loadConfig(configPath)
if err != nil {
return config.Cfg{}, nil, fmt.Errorf("load config: config_path %q: %w", configPath, err)
}
urlSanitizer := log.NewURLSanitizerHook()
urlSanitizer.AddPossibleGrpcMethod(
"CreateRepositoryFromURL",
"FetchRemote",
"UpdateRemoteMirror",
)
errorTruncator := &log.ErrorTruncatorHook{MaxBytes: 1024}
logger, err := log.Configure(
log.NewSyncWriter(os.Stdout),
cfg.Logging.Format,
cfg.Logging.Level,
urlSanitizer,
errorTruncator,
)
if err != nil {
return config.Cfg{}, nil, fmt.Errorf("configuring logger failed: %w", err)
}
if err := cfg.ValidateV2(); err != nil {
logger.Warn(
fmt.Sprintf(
"The current configurations will cause Gitaly to fail to start up in future versions. Please run 'gitaly configuration validate < %s' and fix the errors that are printed.",
configPath,
),
)
}
if undo, err := maxprocs.Set(maxprocs.Logger(func(s string, i ...interface{}) {
logger.Info(fmt.Sprintf(s, i...))
})); err != nil {
logger.WithError(err).Error("failed to set GOMAXPROCS")
undo()
}
sentry.ConfigureSentry(logger, version.GetVersion(), sentry.Config(cfg.Logging.Sentry))
cfg.Prometheus.Configure(logger)
preloadLicenseDatabase(logger)
return cfg, logger, nil
}
// newGitlabAPIClient selects the client that answers access checks and hook
// callbacks. In Graft standalone mode it is the built-in policy client and
// GitLab is never contacted. Otherwise it is the stub client when hooks are
// bypassed for testing, and the GitLab HTTP client in all other cases.
func newGitlabAPIClient(logger log.Logger, cfg config.Cfg, policy *policystore.Store) (gitlab.Client, error) {
if cfg.Graft.Standalone {
logger.Info("graft standalone mode: access checks use the built-in policy, GitLab is not contacted")
return gitlab.NewStandaloneClientWithPolicy(policy, logger), nil
}
if skipHooks, _ := env.GetBool("GITALY_TESTING_NO_GIT_HOOKS", false); skipHooks {
logger.Warn("skipping GitLab API client creation since hooks are bypassed via GITALY_TESTING_NO_GIT_HOOKS")
return gitlab.NewStubClient(), nil
}
httpClient, err := gitlab.NewHTTPClient(logger, cfg.Gitlab, cfg.TLS, cfg.Prometheus)
if err != nil {
return nil, fmt.Errorf("could not create GitLab API client: %w", err)
}
prometheus.MustRegister(httpClient)
return httpClient, nil
}
// newDerivedPool returns the pool that runs the eager derived-data warmers, sized
// from the [graft.derived] configuration. The pool is registered with registerer, so
// that its graft_derived_warm_* metrics are exported.
func newDerivedPool(logger log.Logger, cfg config.Cfg, repoBuilder eager.RepoBuilder, catfileCache catfile.Cache, registerer prometheus.Registerer) *derived.Pool {
pool := derived.NewPool(logger, cfg.Graft.Derived.WorkersOrDefault(), cfg.Graft.Derived.QueueSizeOrDefault(),
eager.WarmEnvOpener(repoBuilder, logger, catfileCache))
registerer.MustRegister(pool)
return pool
}
// newEagerOptions returns the options of the eager Observer from the [graft.derived]
// configuration. Eager derived data and the commit-graph write are enabled only by
// their own keys, and neither runs when derived data is disabled. A configuration
// without any [graft] section therefore enqueues nothing and writes no commit-graph
// outside housekeeping. The listings of new commits are bounded like the warmers
// they feed.
func newEagerOptions(cfg config.Cfg) eager.Options {
return eager.Options{
Eager: cfg.Graft.Derived.Eager && !cfg.Graft.Derived.Disabled,
CommitGraph: cfg.Graft.Derived.CommitGraph && !cfg.Graft.Derived.Disabled,
ListWorkers: cfg.Graft.Derived.WorkersOrDefault(),
ListQueueSize: cfg.Graft.Derived.QueueSizeOrDefault(),
}
}
func preloadLicenseDatabase(logger log.Logger) {
go func() {
// the first call to `licensedb.Detect` could be too long
// https://github.com/go-enry/go-license-detector/issues/13
// this is why we're calling it here to preload license database
// on server startup to avoid long initialization on gRPC
// method handling.
began := time.Now()
licensedb.Preload()
logger.WithField("duration_ms", time.Since(began).Milliseconds()).Info("License database preloaded")
}()
}
func run(appCtx *cli.Command, cfg config.Cfg, logger log.Logger, policySignals chan os.Signal) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// The configuration as the file gave it, before the setup below changes some of it (the
// runtime directory): a reload of the policy compares the file with it.
loaded := cfg
// shutdownCtx is cancelled once the graceful shutdown period has elapsed. It lets
// services terminate work that intentionally ignores the per-request context, such
// as the git-pack-objects processes driven by PackObjectsHookWithSidechannel. Such
// a handler otherwise never returns while its transfer is stuck, which keeps the
// server's GracefulStop (and therefore the force Stop that follows it) from
// completing, wedging shutdown. It is cancelled explicitly below once b.Wait()
// returns, before the force stop and the deferred cleanups run.
shutdownCtx, cancelShutdown := context.WithCancel(context.Background())
defer cancelShutdown()
beganRun := time.Now()
bootstrapSpan, ctx := tracing.StartSpan(ctx, "gitaly-bootstrap", nil)
defer bootstrapSpan.End()
if cfg.RuntimeDir != "" {
if err := config.PruneOldGitalyProcessDirectories(logger, cfg.RuntimeDir); err != nil {
return fmt.Errorf("prune runtime directories: %w", err)
}
}
var err error
cfg, err = config.SetupRuntimeDirectory(cfg, os.Getpid())
if err != nil {
return fmt.Errorf("setup runtime directory: %w", err)
}
cgroupMgr := cgroups.NewManager(cfg.Cgroups, logger, os.Getpid())
loadMonitor := loadmonitor.NewLoadMonitor(loadmonitor.Config{
PollInterval: time.Second * 10,
NotifyTimeout: time.Second * 3,
}, logger, cgroupMgr)
if err = loadMonitor.Start(ctx); err != nil {
return fmt.Errorf("starting load monitor: %w", err)
}
defer loadMonitor.Stop()
prometheus.MustRegister(loadMonitor)
began := time.Now()
if err := cgroupMgr.Setup(); err != nil {
return fmt.Errorf("failed setting up cgroups: %w", err)
}
if cgroupMgr.Ready() {
logger.WithField("duration_ms", time.Since(began).Milliseconds()).Info("finished initializing cgroups")
}
defer func() {
if err := os.RemoveAll(cfg.RuntimeDir); err != nil {
logger.Warn("could not clean up runtime dir")
}
}()
began = time.Now()
if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir, func(string) bool {
return true
}); err != nil {
return fmt.Errorf("unpack auxiliary binaries: %w", err)
}
logger.WithField("duration_ms", time.Since(began).Milliseconds()).Info("finished unpacking auxiliary binaries")
began = time.Now()
b, err := bootstrap.NewBootstrap(ctx, logger, promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "gitaly_connections_total",
Help: "Total number of connections to Gitaly",
},
[]string{"type"},
))
if err != nil {
return fmt.Errorf("init bootstrap: %w", err)
}
logger.WithField("duration_ms", time.Since(began).Milliseconds()).Info("finished initializing bootstrap")
began = time.Now()
gitCmdFactory, cleanup, err := gitcmd.NewExecCommandFactory(cfg, logger, gitcmd.WithCgroupsManager(cgroupMgr))
if err != nil {
return fmt.Errorf("creating Git command factory: %w", err)
}
defer cleanup()
logger.WithField("duration_ms", time.Since(began).Milliseconds()).Info("finished initializing command factory")
logger.WithField("binary_path", gitCmdFactory.GetExecutionEnvironment(ctx).BinaryPath).Info("using Git binary")
began = time.Now()
gitVersion, err := gitCmdFactory.GitVersion(ctx)
if err != nil {
return fmt.Errorf("git version detection: %w", err)
}
logger.WithField("duration_ms", time.Since(began).Milliseconds()).Info("finished detecting git version")
logger.WithField("version", gitVersion).Info("using Git version")
registry := backchannel.NewRegistry()
transactionManager := transaction.NewManager(cfg, logger, registry)
prometheus.MustRegister(transactionManager)
locator := config.NewLocator(cfg)
repoCounter := counter.NewRepositoryCounter(cfg.Storages)
prometheus.MustRegister(repoCounter)
prometheus.MustRegister(gitCmdFactory)
conns := client.NewPool(
client.WithDialer(client.HealthCheckDialer(
func(ctx context.Context, address string, opts []grpc.DialOption) (*grpc.ClientConn, error) {
return client.New(ctx, address, client.WithGrpcOptions(opts))
},
)),
client.WithDialOptions(
grpc.WithStatsHandler(tracing.NewGRPCClientStatsHandler()),
client.UnaryInterceptor(),
client.StreamInterceptor(),
gitalyclient.WithGitalyDNSResolver(gitalyclient.DefaultDNSResolverBuilderConfig()),
),
)
defer func() {
_ = conns.Close()
}()
catfileCache := catfile.NewCache(cfg)
defer catfileCache.Stop()
prometheus.MustRegister(catfileCache)
localrepoFactory := localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache)
diskCache := cache.New(cfg, locator, logger)
prometheus.MustRegister(diskCache)
if err := diskCache.StartWalkers(); err != nil {
return fmt.Errorf("disk cache walkers: %w", err)
}
// List of tracking adaptive limits. They will be calibrated by the adaptive calculator
adaptiveLimits := []limiter.AdaptiveLimiter{}
perRPCLimits, setupPerRPCConcurrencyLimiters := limithandler.WithConcurrencyLimiters(cfg)
for _, concurrency := range cfg.Concurrency {
// Connect adaptive limits to the adaptive calculator
if concurrency.Adaptive {
if limit, ok := perRPCLimits.Authenticated[concurrency.RPC]; ok {
adaptiveLimits = append(adaptiveLimits, limit)
}
}
if concurrency.Global.Adaptive {
if globalLimit, ok := perRPCLimits.AuthenticatedGlobal[concurrency.RPC]; ok {
adaptiveLimits = append(adaptiveLimits, globalLimit)
}
}
if concurrency.Unauthenticated.Adaptive {
if unauthLimit, ok := perRPCLimits.Unauthenticated[concurrency.RPC]; ok {
adaptiveLimits = append(adaptiveLimits, unauthLimit)
}
}
if concurrency.Unauthenticated.Global.Adaptive {
if globalLimit, ok := perRPCLimits.UnauthenticatedGlobal[concurrency.RPC]; ok {
adaptiveLimits = append(adaptiveLimits, globalLimit)
}
}
}
perRPCLimitHandler := limithandler.New(
cfg,
limithandler.LimitConcurrencyByRepo,
setupPerRPCConcurrencyLimiters,
)
prometheus.MustRegister(perRPCLimitHandler)
packObjectLimit := newPackObjectsAdaptiveLimit("packObjects", cfg.PackObjectsLimiting.ConcurrencyLimits)
if cfg.PackObjectsLimiting.Adaptive {
adaptiveLimits = append(adaptiveLimits, packObjectLimit)
}
packObjectsMonitor := limiter.NewPackObjectsConcurrencyMonitor(
cfg.Prometheus.GRPCLatencyBuckets,
"authenticated",
)
packObjectsLimiterAuthenticated := limiter.NewConcurrencyLimiter(
packObjectLimit,
cfg.PackObjectsLimiting.QueueMax(),
cfg.PackObjectsLimiting.MaxQueueWait.Duration(),
packObjectsMonitor,
)
prometheus.MustRegister(packObjectsMonitor)
var packObjectsLimiter limiter.Limiter = packObjectsLimiterAuthenticated
if cfg.PackObjectsLimiting.Unauthenticated.IsSet() {
unauthLimits := cfg.PackObjectsLimiting.Unauthenticated
packObjectLimitUnauth := newPackObjectsAdaptiveLimit(
"packObjects-unauthenticated", unauthLimits.ConcurrencyLimits,
)
if unauthLimits.Adaptive {
adaptiveLimits = append(adaptiveLimits, packObjectLimitUnauth)
}
packObjectsMonitorUnauth := limiter.NewPackObjectsConcurrencyMonitor(
cfg.Prometheus.GRPCLatencyBuckets,
"unauthenticated",
)
prometheus.MustRegister(packObjectsMonitorUnauth)
// Anonymous traffic may additionally be bounded across the node as a whole, which
// keeps its budget from growing with the number of addresses it arrives from.
var opts []limiter.Option
if unauthLimits.Global.IsSet() {
packObjectLimitGlobal := newPackObjectsAdaptiveLimit(
"packObjects-unauthenticated-global", unauthLimits.Global,
)
if unauthLimits.Global.Adaptive {
adaptiveLimits = append(adaptiveLimits, packObjectLimitGlobal)
}
packObjectsMonitorGlobal := limiter.NewPackObjectsConcurrencyMonitor(
cfg.Prometheus.GRPCLatencyBuckets,
"unauthenticated-global",
)
prometheus.MustRegister(packObjectsMonitorGlobal)
opts = append(opts, limiter.WithGlobalLimit(
packObjectLimitGlobal,
unauthLimits.Global.MaxQueueSize,
unauthLimits.Global.MaxQueueWait.Duration(),
packObjectsMonitorGlobal,
))
}
packObjectsLimiterUnauthenticated := limiter.NewConcurrencyLimiter(
packObjectLimitUnauth,
unauthLimits.MaxQueueSize,
unauthLimits.MaxQueueWait.Duration(),
packObjectsMonitorUnauth,
opts...,
)
// Use DualLimiter to switch between authenticated and unauthenticated limiters
packObjectsLimiter = limiter.NewDualLimiter(packObjectsLimiterAuthenticated, packObjectsLimiterUnauthenticated)
}
// Enable the adaptive calculator only if there is any limit needed to be adaptive.
if len(adaptiveLimits) > 0 {
calcCfg := limiter.Config{
CalibrationInterval: limiter.DefaultCalibrateFrequency,
CPUThreshold: cfg.AdaptiveLimiting.CPUThrottledThreshold,
MemoryThreshold: cfg.AdaptiveLimiting.MemoryThreshold,
PSIConfig: cfg.AdaptiveLimiting.PSIPressure,
}
adaptiveCalculator := limiter.NewAdaptiveCalculator(calcCfg, logger, loadMonitor, adaptiveLimits)
prometheus.MustRegister(adaptiveCalculator)
stop, err := adaptiveCalculator.Start(ctx)
if err != nil {
logger.WithError(err).Warn("error starting adaptive limiter calculator")
}
defer stop()
}
mvccCacheFactory := gitmvcc.NewCacheFactory(cfg.RuntimeDir, locator)
defer func() { _ = mvccCacheFactory.Close(ctx) }()
prometheus.MustRegister(mvccCacheFactory)
// The proposals of the land expire in the housekeeping pass too, so that those of idle and
// deleted branches go (PLAN10 14). proposal_ttl was validated with the configuration.
proposalTTL, err := cfg.Graft.Land.Resolver.ProposalTTLOrDefault()
if err != nil {
return fmt.Errorf("graft land resolver: %w", err)
}
housekeepingManager := housekeepingmgr.New(cfg.Prometheus, logger, transactionManager, nil,
housekeepingmgr.WithMVCCCacheSupport(mvccCacheFactory), housekeepingmgr.WithGraftProposalTTL(proposalTTL))
prometheus.MustRegister(housekeepingManager)
housekeepingMiddlewareConfig := housekeepingmw.DefaultMiddlewareConfig()
housekeepingMiddleware := housekeepingmw.NewHousekeepingMiddleware(logger, protoregistry.GitalyProtoPreregistered, localrepoFactory, housekeepingManager, housekeepingMiddlewareConfig)
// Stop cancels any in-flight housekeeping workers before waiting for them. The workers are
// detached from their request context, so without cancellation they keep optimizing the
// repository until completion and can block shutdown for minutes after the grace period
// expired and the servers have been force-stopped.
defer housekeepingMiddleware.Stop()
bm := burdenmonitor.New(logger)
bm.StartPoller(ctx)
if cfg.LoadShedder.Enabled {
loadShedder, err := burdenmonitor.NewLoadShedder(
burdenmonitor.LoadShedderConfig{PSI: cfg.AdaptiveLimiting.PSIPressure},
logger, loadMonitor, bm,
)
if err != nil {
return fmt.Errorf("creating load shedder: %w", err)
}
loadShedder.Start(ctx)
}
mvccMiddleware := mvcc.NewMvccInterceptor(mvccCacheFactory, logger, protoregistry.GitalyProtoPreregistered, locator)
gitalyServerFactory := server.NewGitalyServerFactory(
cfg,
logger,
registry,
diskCache,
[]*limithandler.LimiterMiddleware{perRPCLimitHandler},
housekeepingMiddleware,
server.TransactionMiddleware{},
mvccMiddleware,
)
// The standalone access policy is held by one store, which every reader decides with and
// SIGUSR1 reloads from the configuration file (doc/graft/PLAN12.md).
graftPolicy := policystore.New(cfg.Graft.Policy)
prometheus.MustRegister(graftPolicy)
stopPolicyReload := reloadPolicyOnSignal(logger, appCtx.Args().First(), loaded, graftPolicy, policySignals)
defer stopPolicyReload()
gitlabClient, err := newGitlabAPIClient(logger, cfg, graftPolicy)
if err != nil {
return err
}
// Eager derived data and commit-graph writes run after committed reference
// updates, see internal/graft/eager. The warmers run in one bounded pool.
repoBuilder := func(repo storage.Repository) *localrepo.Repo {
return localrepo.New(logger, locator, gitCmdFactory, catfileCache, repo)
}
derivedPool := newDerivedPool(logger, cfg, repoBuilder, catfileCache, prometheus.DefaultRegisterer)
go derivedPool.Run(ctx)
defer derivedPool.Close()
eagerObserver := eager.NewObserver(ctx, logger,
newEagerOptions(cfg),
eager.Deps{
Enqueuer: derivedPool,
ListNewCommits: eager.NewCommitsLister(repoBuilder),
WriteCommitGraph: eager.CommitGraphWriterFor(repoBuilder, housekeepingManager), // serialized with OptimizeRepository
},
)
defer eagerObserver.Close()
prometheus.MustRegister(eagerObserver)
// The Graft storage core keeps the repository state in a database plus object storage
// (doc/graft/STORAGE.md). It is nil when [graft.storage] mode is off.
storageCore, err := corestoreopen.Open(ctx, cfg.Graft.Storage)
if err != nil {
return fmt.Errorf("graft storage core: %w", err)
}
defer func() { _ = storageCore.Close() }()
// The review roots are written and swept in every mode, also without the
// storage core.
prometheus.MustRegister(reviewroots.DefaultSweeper)
hookManagerOpts := []hook.ManagerOption{hook.WithEagerObserver(eagerObserver), hook.WithPolicyStore(graftPolicy)}
var (
storageSyncer corestore.Freshener
storageGate *gate.Gate
)
if storageCore.Enabled() {
if storageCore.Authoritative() {
if err := openLocalReferenceWrites(storageCore, locator, logger); err != nil {
return err
}
}
syncer := corestoresyncer.New(storageCore, logger, locator, localrepoFactory, gitCmdFactory)
prometheus.MustRegister(syncer)
storageSyncer = syncer
storageVoter := voter.New(storageCore, logger, localrepoFactory, locator, syncer)
prometheus.MustRegister(storageVoter)
hookManagerOpts = append(hookManagerOpts, hook.WithStorageVoter(storageVoter))
if storageCore.Authoritative() {
// Before any listener opens: a repository whose references may be torn
// by a kernel restart is moved aside, so nothing reads it before it is
// hydrated again (doc/graft/PLAN7.md, LOCAL REFERENCE WRITES RULE).
storageNames := make([]string, 0, len(cfg.Storages))
for _, s := range cfg.Storages {
storageNames = append(storageNames, s.Name)
}
// The build directories of hydrations of a process that died are
// removed; those of a process that runs (the old one of a graceful
// upgrade) hold their lock and are kept. A failure is logged and
// counted: it never stops the server.
syncer.CleanBuilds(ctx, storageNames)
stopLocalReferences, err := syncer.StartLocalReferences(ctx, storageNames)
if err != nil {
return err
}
// Deferred here, so it runs after the servers stopped: a last durable step
// settles every repository whose writes ended.
defer stopLocalReferences()
// Runs before it (deferred later): the seeds in the background end
// before the last durable step.
defer syncer.StopSeeds()
if cfg.Graft.Storage.MaintenanceAuto() {
stopMaintenance, err := startStorageMaintenance(ctx, cfg, storageCore, syncer, locator, localrepoFactory, logger, storageNames)
if err != nil {
return err
}
defer stopMaintenance()
}
storageGate = gate.New(storageCore, syncer, logger, locator, localrepoFactory, protoregistry.GitalyProtoPreregistered)
}
logger.WithField("mode", string(storageCore.Mode)).WithField("node_id", storageCore.NodeID).Info("graft storage core enabled")
}
postReceiveRegistry := hook.NewPostReceiveRegistry()
hookManager := hook.NewManager(
cfg,
locator,
logger,
gitCmdFactory,
cgroupMgr,
transactionManager,
gitlabClient,
postReceiveRegistry,
hookManagerOpts...,
)
prometheus.MustRegister(hookManager)
updaterWithHooks := updateref.NewUpdaterWithHooks(cfg, logger, locator, hookManager, gitCmdFactory, catfileCache)
packObjectStreamCache := streamcache.New(cfg.PackObjectsCache, logger)
archiveStreamCache := streamcache.New(cfg.ArchiveCache, logger)
var backupSink *backup.Sink
var backupLocator backup.Locator
if cfg.Backup.GoCloudURL != "" {
var err error
backupSink, err = backup.ResolveSink(ctx, cfg.Backup.GoCloudURL, backup.WithBufferSize(cfg.Backup.BufferSize))
if err != nil {
return fmt.Errorf("resolve backup sink: %w", err)
}
backupLocator = backup.NewLocator(backupSink)
}
var poolMetadataStore relational.PoolStore
if cfg.PoolMetadata.DatabasePath != "" {
var err error
poolMetadataStore, err = relational.NewSQLitePoolStore(cfg.PoolMetadata.DatabasePath)
if err != nil {
return fmt.Errorf("create pool metadata store: %w", err)
}
defer func() { _ = poolMetadataStore.Close() }()
logger.Info(fmt.Sprintf("pool metadata store configured: %s", cfg.PoolMetadata.DatabasePath))
}
objectPoolStateManager := relational.NewObjectPoolStateManager(poolMetadataStore)
var bundleURIManager *bundleuri.GenerationManager
if cfg.BundleURI.GoCloudURL != "" {
bundleURISink, err := bundleuri.NewSink(ctx, cfg.BundleURI.GoCloudURL)
if err != nil {
return fmt.Errorf("create bundle-URI sink: %w", err)
}
// The manager created here merely to have a non-nil manager in order to use
// the SignedURL() method on it. It is not used, yet, to generate bundles
// based on this configuration.
// Further tests and analysis would be required to come up with the
// appropriate configuration. This will be done once we are ready to use this manager
// to generate bundles.
maxBundleAge := time.Hour * 24
interval := time.Minute
maxConcurrent := 5
threshold := 5
bundleGenerationStrategy, err := bundleuri.NewOccurrenceStrategy(logger, threshold, interval, maxConcurrent, maxBundleAge)
if err != nil {
return fmt.Errorf("error creating bundle generation strategy: %w", err)
}
prometheus.MustRegister(bundleGenerationStrategy)
stop := bundleGenerationStrategy.Start(ctx)
defer stop()
bundleURIManager, err = bundleuri.NewGenerationManager(ctx, bundleURISink, logger, nil, bundleGenerationStrategy)
if err != nil {
return fmt.Errorf("error creating bundle manager: %w", err)
}
logger.Info(fmt.Sprintf("bundle-uri bucket configured: %s", cfg.BundleURI.GoCloudURL))
prometheus.MustRegister(bundleURIManager)
}
// The land batching lanes are shared by the operations servers of every listener, so that the
// requests for one branch are batched in one lane. They are closed after the servers stopped,
// once the shutdown context has ended the batches that are still running.
landLaneCfg, err := landlane.ConfigFromGraft(cfg.Graft.Land.Batch)
if err != nil {
return fmt.Errorf("graft land batching: %w", err)
}
landLanes := landlane.NewShared(landLaneCfg)
prometheus.MustRegister(landLanes)
defer landLanes.Close()
// The conflict resolver is built once and shared by the operations servers of every
// listener, so that its check slots bound the process (check_concurrency) and a resolver
// that cannot be built is one error, which makes every land that needs it fail closed.
graftResolver := resolver.NewShared(cfg.Graft.Land.Resolver)
if graftResolver.Err != nil {
logger.WithError(graftResolver.Err).Error("graft: the conflict resolver of land cannot be built: lands that need it fail")
}
for _, c := range []starter.Config{
{Name: starter.Unix, Addr: cfg.SocketPath, HandoverOnUpgrade: true},
{Name: starter.Unix, Addr: cfg.InternalSocketPath(), HandoverOnUpgrade: false},
{Name: starter.TCP, Addr: cfg.ListenAddr, HandoverOnUpgrade: true},
{Name: starter.TLS, Addr: cfg.TLSListenAddr, HandoverOnUpgrade: true},
} {
if c.Addr == "" {
continue
}
var srv *grpc.Server
opts := []server.Option{
server.WithUnaryInterceptor(bm.UnaryInterceptor()),
server.WithStreamInterceptor(bm.StreamInterceptor()),
}
if storageGate != nil {
opts = append(opts,
server.WithUnaryInterceptor(storageGate.UnaryInterceptor()),
server.WithStreamInterceptor(storageGate.StreamInterceptor()),
)
}
if c.HandoverOnUpgrade {
srv, err = gitalyServerFactory.CreateExternal(c.IsSecure(), opts...)
if err != nil {
return fmt.Errorf("create external gRPC server: %w", err)
}
} else {
srv, err = gitalyServerFactory.CreateInternal(opts...)
if err != nil {
return fmt.Errorf("create internal gRPC server: %w", err)
}
}
setup.RegisterAll(srv, &service.Dependencies{
Logger: logger,
Cfg: cfg,
GitalyHookManager: hookManager,
TransactionManager: transactionManager,
StorageLocator: locator,
ClientPool: conns,
GitCmdFactory: gitCmdFactory,
CgroupsManager: cgroupMgr,
CatfileCache: catfileCache,
DiskCache: diskCache,
PackObjectsCache: packObjectStreamCache,
PackObjectsLimiter: packObjectsLimiter,
RepositoryCounter: repoCounter,
UpdaterWithHooks: updaterWithHooks,
HousekeepingManager: housekeepingManager,
BackupSink: backupSink,
BackupLocator: backupLocator,
LocalRepositoryFactory: localrepoFactory,
BundleURIManager: bundleURIManager,
ArchiveCache: archiveStreamCache,
PoolMetadataStore: poolMetadataStore,
GitlabClient: gitlabClient,
GraftPolicy: graftPolicy,
ObjectPoolStateManager: objectPoolStateManager,
PostReceiveRegistry: postReceiveRegistry,
GraftLandLanes: landLanes,
GraftResolver: graftResolver,
GraftStorage: storageCore,
GraftStorageSyncer: storageSyncer,
ShutdownContext: shutdownCtx,
})
b.RegisterStarter(starter.New(c, srv, logger))
}
if addr := cfg.PrometheusListenAddr; addr != "" {
b.RegisterStarter(func(listen bootstrap.ListenFunc, _ chan<- error, _ *prometheus.CounterVec) error {
l, err := listen("tcp", addr)
if err != nil {
return err
}
logger.WithField("address", addr).Info("starting prometheus listener")
go func() {
opts := []monitoring.Option{
monitoring.WithListener(l),
monitoring.WithBuildExtraLabels(
map[string]string{"git_version": gitVersion},
),
}
if buildInfo, ok := debug.ReadBuildInfo(); ok {
opts = append(opts, monitoring.WithGoBuildInformation(buildInfo))
}
if err := monitoring.Start(opts...); err != nil {
logger.WithError(err).Error("Unable to serve prometheus")
}
}()
return nil
})
}
for _, shard := range cfg.Storages {
if err := mdfile.WriteMetadataFile(ctx, shard.Path); err != nil {
// TODO should this be a return? https://gitlab.com/gitlab-org/gitaly/issues/1893
logger.WithError(err).Error("Unable to write gitaly metadata file")
}
}
// When cgroups are configured, we create a directory structure each
// time a gitaly process is spawned. Look through the hierarchy root
// to find any cgroup directories that belong to old gitaly processes
// and remove them.
cgroups.StartPruningOldCgroups(cfg.Cgroups, logger)
repoCounter.StartCountingRepositories(ctx, locator, logger)
tempdir.StartCleaning(logger, locator, cfg.Storages, time.Hour)
if err := b.Start(); err != nil {
return fmt.Errorf("unable to start the bootstrap: %w", err)
}
bootstrapSpan.End()
// There are a few goroutines running async tasks that may still be in progress (i.e. preloading the license
// database), but this is a close enough indication of startup latency.
logger.WithField("duration_ms", time.Since(beganRun).Milliseconds()).Info("Started Gitaly")
if !cfg.DailyMaintenance.IsDisabled() {
shutdownWorkers, err := maintenance.StartWorkers(
ctx,
logger,
maintenance.DailyOptimizationWorker(cfg, dailyOptimizer(
func(logger log.Logger, repo storage.Repository) *localrepo.Repo {
return localrepo.New(logger, locator, gitCmdFactory, catfileCache, repo)
},
func(ctx context.Context, logger log.Logger, local *localrepo.Repo, repo storage.Repository) error {
return housekeepReviewRoots(ctx, logger, storageCore, storageSyncer, local, repo)
},
func(ctx context.Context, local *localrepo.Repo) error {
return housekeepingManager.OptimizeRepository(ctx, local)
},
)),
)
if err != nil {
return fmt.Errorf("initialize auxiliary workers: %w", err)
}
defer shutdownWorkers()
}
gracefulStopTicker := helper.NewTimerTicker(cfg.GracefulRestartTimeout.Duration())
defer gracefulStopTicker.Stop()
logger.WithField("duration_s", cfg.GracefulRestartTimeout.Duration().String()).Info("Gitaly graceful stop timeout duration")
waitErr := b.Wait(gracefulStopTicker, gitalyServerFactory.GracefulStop)
logger.WithField("error_message", waitErr).Info("Wait has returned with error")
// The grace period has elapsed. Cancel the shutdown context so that services
// terminate work that intentionally ignores the per-request context (such as
// git-pack-objects driven by PackObjectsHookWithSidechannel). Such a handler
// otherwise keeps running, so the in-flight GracefulStop above never finishes
// its handlersWG wait; because it holds the gRPC server mutex while waiting,
// the force Stop below cannot even acquire the mutex, and shutdown wedges until
// the handler eventually returns on its own.
cancelShutdown()
// Arm a watchdog that dumps the stacks of any goroutines still holding up
// shutdown, so a wedged shutdown is observable rather than a silent
// multi-minute hang. It only logs; it does not change shutdown behavior. It is
// intentionally not disarmed: a clean shutdown exits before the timeout, which
// kills the watchdog goroutine. See internal/bootstrap/shutdown_watchdog.go.
bootstrap.StartShutdownWatchdog(logger, bootstrap.ShutdownWatchdogTimeout(cfg.GracefulRestartTimeout.Duration()))
// Always force-stop servers immediately after Wait returns. If the grace
// period expired or a force shutdown was requested, this ensures in-flight
// connections are killed before deferred functions run, preventing them
// from blocking on operations that will never complete. If GracefulStop
// already completed cleanly, this is a no-op.
gitalyServerFactory.Stop()
return waitErr
}
// dailyOptimizer is the job of the daily maintenance for one repository: the backstop of the
// review roots, in every mode, then the housekeeping that prunes unreferenced objects
// (doc/graft/PLAN7.md, REVIEW ROOTS RULE). The commits of an open review written without its
// keep-around references are referenced before a prune could take them, so a failed backstop
// skips the housekeeping of the repository.
func dailyOptimizer(
newLocal func(log.Logger, storage.Repository) *localrepo.Repo,
backstop func(context.Context, log.Logger, *localrepo.Repo, storage.Repository) error,
optimize func(context.Context, *localrepo.Repo) error,
) maintenance.Optimizer {
return maintenance.OptimizerFunc(func(ctx context.Context, logger log.Logger, repo storage.Repository) error {
local := newLocal(logger, repo)
if err := backstop(ctx, logger, local, repo); err != nil {
return err
}
return optimize(ctx, local)
})
}
// housekeepReviewRoots runs the backstop of the review roots before the daily
// housekeeping of a repository: it writes the keep-around references that
// open reviews lack and deletes those no open review uses (in authoritative
// mode after catching the repository up). Open reviews whose commits are lost
// are logged with their IDs and commits; the counts are metrics of
// reviewroots.DefaultSweeper.
func housekeepReviewRoots(ctx context.Context, logger log.Logger, core *corestore.Core, fresh corestore.Freshener, local *localrepo.Repo, repo storage.Repository) error {
backfill, sweep, err := reviewroots.DefaultSweeper.Housekeep(ctx, core, fresh, local, repo)
for _, missing := range backfill.Missing {
logger.WithFields(log.Fields{
"review_id": missing.ReviewID,
"commit_id": missing.Commit,
}).Warn("review roots: a commit of an open review is not in the repository; the review can only be closed")
}
if backfill.Protected > 0 || sweep.Deleted > 0 {
logger.WithFields(log.Fields{
"backfilled": backfill.Protected,
"deleted": sweep.Deleted,
}).Info("review roots: housekeeping")
}
if err != nil {
return fmt.Errorf("review roots: %w", err)
}
return nil
}
// openLocalReferenceWrites sets up the admission and the deferred durability
// of the local reference writes of the storage core in authoritative mode
// (doc/graft/PLAN7.md, LOCAL REFERENCE WRITES RULE). Without the kernel's boot
// identity (not Linux) the durability is not deferred: every reference write
// is made durable by git, as in the other modes.
func openLocalReferenceWrites(core *corestore.Core, locator storage.Locator, logger log.Logger) error {
lockTimeout := gitcmd.ReftableLockTimeoutMilliseconds * time.Millisecond
core.RefAdmission = localrefs.NewAdmission(lockTimeout, lockTimeout)
prometheus.MustRegister(core.RefAdmission)
durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle,
localrefs.WithWatermark(gitops.WatermarkPath))
if err != nil {
return fmt.Errorf("graft storage core: %w", err)
}
if durability == nil {
logger.Info("graft storage core: the kernel exposes no boot identity, every reference write is made durable by git")
return nil
}
core.RefDurability = durability
prometheus.MustRegister(durability)
return nil
}
// startStorageMaintenance starts the background checkpoints and garbage
// collection of the storage core ([graft.storage] checkpoints, on by default,
// and gc, opt-in; authoritative mode only; doc/graft/PLAN7.md, "Background
// maintenance").
func startStorageMaintenance(ctx context.Context, cfg config.Cfg, core *corestore.Core, fresh corestore.Freshener, locator storage.Locator, repoFactory localrepo.Factory, logger log.Logger, storageNames []string) (func(), error) {
interval, err := cfg.Graft.Storage.MaintenanceIntervalOrDefault()
if err != nil {
return nil, err
}
leaseTTL, err := cfg.Graft.Storage.LeaseTTLOrDefault()
if err != nil {
return nil, err
}
runner := storagemaintenance.New(core,
checkpoint.New(core, logger, locator, repoFactory, fresh),
gc.New(core, logger),
fresh, logger,
storagemaintenance.Config{
Interval: interval,
Checkpoints: cfg.Graft.Storage.CheckpointsAuto(),
CheckpointChangesets: cfg.Graft.Storage.CheckpointChangesetsOrDefault(),
CheckpointPackBytes: cfg.Graft.Storage.CheckpointPackBytesOrDefault(),
Collect: cfg.Graft.Storage.GCAuto(),
LeaseTTL: leaseTTL,
},
storageNames,
)
prometheus.MustRegister(runner)
logger.WithField("interval", interval.String()).
WithField("checkpoints", cfg.Graft.Storage.CheckpointsAuto()).
WithField("gc", cfg.Graft.Storage.GCAuto()).
Info("graft storage core: background maintenance enabled")
return runner.Start(ctx), nil
}
// newPackObjectsAdaptiveLimit builds the adaptive limit backing a pack objects limiter. The limit
// only moves when the configuration asks for an adaptive one, and is otherwise pinned to the
// configured concurrency.
func newPackObjectsAdaptiveLimit(name string, limits config.ConcurrencyLimits) *limiter.AdaptiveLimit {
if !limits.Adaptive {
return limiter.NewAdaptiveLimit(name, limiter.AdaptiveSetting{
Initial: limits.Concurrency(),
})
}
return limiter.NewAdaptiveLimit(name, limiter.AdaptiveSetting{
Initial: limits.InitialLimit,
Max: limits.MaxLimit,
Min: limits.MinLimit,
BackoffFactor: limiter.DefaultBackoffFactor,
})
}