Git designed for AI agents.
package praefect
import (
"context"
"errors"
"fmt"
"net"
"testing"
"time"
gitalycfgauth "github.com/feder-cr/git-ai/internal/gitaly/config/auth"
"github.com/feder-cr/git-ai/internal/gitaly/server/auth"
"github.com/feder-cr/git-ai/internal/grpc/client"
"github.com/feder-cr/git-ai/internal/grpc/protoregistry"
"github.com/feder-cr/git-ai/internal/log"
"github.com/feder-cr/git-ai/internal/praefect/config"
"github.com/feder-cr/git-ai/internal/praefect/datastore"
"github.com/feder-cr/git-ai/internal/praefect/datastore/glsql"
"github.com/feder-cr/git-ai/internal/praefect/nodes"
"github.com/feder-cr/git-ai/internal/praefect/service"
"github.com/feder-cr/git-ai/internal/praefect/transactions"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/feder-cr/git-ai/internal/testhelper/promtest"
"github.com/feder-cr/git-ai/internal/testhelper/testdb"
"github.com/feder-cr/git-ai/internal/testhelper/testserver"
"github.com/feder-cr/git-ai/proxy"
"github.com/stretchr/testify/require"
correlation "gitlab.com/gitlab-org/labkit/correlation/grpc"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
healthpb "google.golang.org/grpc/health/grpc_health_v1"
)
// BuildOptions is a set of configurations options that can be set to configure praefect service.
type BuildOptions struct {
// WithQueue sets an implementation of the replication queue to use by praefect service.
WithQueue datastore.ReplicationEventQueue
// WithTxMgr sets the transaction manager to use by praefect service.
WithTxMgr *transactions.Manager
// WithBackends sets a callback that is triggered during initialization.
WithBackends func([]*config.VirtualStorage) []testhelper.Cleanup
// WithAnnotations sets a proto-registry to use by praefect service.
WithAnnotations *protoregistry.Registry
// WithLogger sets a logger to use by praefect service.
WithLogger log.Logger
// WithNodeMgr sets an implementation of the node manager to use by praefect service.
WithNodeMgr nodes.Manager
// WithRepoStore sets an implementation of the repositories store to use by praefect service.
WithRepoStore datastore.RepositoryStore
// WithAssignmentStore sets an implementation of the repositories store to use by praefect service.
WithAssignmentStore AssignmentStore
// WithConnections sets a set of connections to gitalies.
WithConnections Connections
// WithPrimaryGetter sets an implementation of the primary node getter to use by praefect service.
WithPrimaryGetter PrimaryGetter
// WithRouter sets an implementation of the request router to use by praefect service.
WithRouter Router
// WithChecks sets a list of check to run when ReadinessCheck RPC is called.
WithChecks []service.CheckFunc
}
// WithMockBackends mocks backends with a set of passed in functions that know to register a gRPC
// server.
func WithMockBackends(tb testing.TB, backendRegistrars map[string]func(*grpc.Server)) func([]*config.VirtualStorage) []testhelper.Cleanup {
return func(virtualStorages []*config.VirtualStorage) []testhelper.Cleanup {
var cleanups []testhelper.Cleanup
for _, vs := range virtualStorages {
require.Equal(tb, len(backendRegistrars), len(vs.Nodes),
"mock server count doesn't match config nodes")
for i, node := range vs.Nodes {
backendRegistrar, ok := backendRegistrars[node.Storage]
require.True(tb, ok, "missing server registrator for node %s", node.Storage)
backendAddr, cleanup := newMockDownstream(tb, node.Token, backendRegistrar)
cleanups = append(cleanups, cleanup)
node.Address = backendAddr
vs.Nodes[i] = node
}
}
return cleanups
}
}
func defaultQueue(tb testing.TB, qc glsql.Querier) datastore.ReplicationEventQueue {
return datastore.NewPostgresReplicationEventQueue(qc)
}
func defaultTxMgr(conf config.Config, logger log.Logger, repoWriteLockMgr datastore.WriteLockManager) *transactions.Manager {
return transactions.NewManager(conf, logger, repoWriteLockMgr)
}
func defaultNodeMgr(tb testing.TB, conf config.Config, rs datastore.RepositoryStore) nodes.Manager {
nodeMgr, err := nodes.NewManager(testhelper.SharedLogger(tb), conf, nil, rs, promtest.NewMockHistogramVec(), protoregistry.GitalyProtoPreregistered, nil, nil, nil)
require.NoError(tb, err)
nodeMgr.Start(0, time.Hour)
tb.Cleanup(nodeMgr.Stop)
return nodeMgr
}
func defaultRepoStore(conf config.Config) datastore.RepositoryStore {
return datastore.MockRepositoryStore{}
}
func listenAvailPort(tb testing.TB) (net.Listener, int) {
ctx := testhelper.Context(tb)
lc := net.ListenConfig{}
listener, err := lc.Listen(ctx, "tcp", "localhost:0")
require.NoError(tb, err)
return listener, listener.Addr().(*net.TCPAddr).Port
}
func dialLocalPort(tb testing.TB, ctx context.Context, port int, backend bool) *grpc.ClientConn {
opts := []grpc.DialOption{
grpc.WithUnaryInterceptor(correlation.UnaryClientCorrelationInterceptor()),
grpc.WithStreamInterceptor(correlation.StreamClientCorrelationInterceptor()),
}
if backend {
opts = append(
opts,
grpc.WithDefaultCallOptions(grpc.ForceCodec(proxy.NewCodec())),
)
}
cc, err := client.New(
ctx,
fmt.Sprintf("tcp://localhost:%d", port),
client.WithGrpcOptions(opts),
)
require.NoError(tb, err)
return cc
}
func newMockDownstream(tb testing.TB, token string, registerService func(*grpc.Server)) (string, func()) {
srv := grpc.NewServer(grpc.UnaryInterceptor(auth.UnaryServerInterceptor(gitalycfgauth.Config{Token: token})))
registerService(srv)
healthpb.RegisterHealthServer(srv, health.NewServer())
// client to backend service
lis, port := listenAvailPort(tb)
errQ := make(chan error)
go func() {
errQ <- srv.Serve(lis)
}()
cleanup := func() {
srv.GracefulStop()
lis.Close()
// If the server is shutdown before Serve() is called on it
// the Serve() calls will return the ErrServerStopped
if err := <-errQ; err != nil && !errors.Is(err, grpc.ErrServerStopped) {
require.NoError(tb, err)
}
}
return fmt.Sprintf("tcp://localhost:%d", port), cleanup
}
type noopBackoffFactory struct{}
func (noopBackoffFactory) Create() (Backoff, BackoffReset) {
return func() time.Duration {
return 0
}, func() {}
}
func startProcessBacklog(ctx context.Context, replMgr ReplMgr) <-chan struct{} {
done := make(chan struct{})
go func() {
defer close(done)
replMgr.ProcessBacklog(ctx, noopBackoffFactory{})
}()
return done
}
// RunPraefectServer starts praefect service based on the passed in configuration and options.
// The caller is responsible to call returned testhelper.Cleanup in order to stop the service
// and release all acquired resources.
// The function should be used only for testing purposes and not as part of the production code.
func RunPraefectServer(
tb testing.TB,
ctx context.Context,
conf config.Config,
opt BuildOptions,
) (*grpc.ClientConn, *grpc.Server, testhelper.Cleanup) {
var cleanups []testhelper.Cleanup
db := testdb.New(tb)
ctx, cancel := context.WithCancel(ctx)
if opt.WithLogger == nil {
opt.WithLogger = testhelper.SharedLogger(tb)
}
if opt.WithQueue == nil {
opt.WithQueue = defaultQueue(tb, db)
}
if opt.WithRepoStore == nil {
opt.WithRepoStore = defaultRepoStore(conf)
}
if opt.WithTxMgr == nil {
repoWriteLockMgr := datastore.NewRepoReferenceWriteLockManager(ctx, db, testdb.GetConfig(tb, db.Name), opt.WithLogger)
opt.WithTxMgr = defaultTxMgr(conf, opt.WithLogger, repoWriteLockMgr)
}
if opt.WithBackends != nil {
cleanups = append(cleanups, opt.WithBackends(conf.VirtualStorages)...)
}
if opt.WithAnnotations == nil {
opt.WithAnnotations = protoregistry.GitalyProtoPreregistered
}
if opt.WithNodeMgr == nil {
opt.WithNodeMgr = defaultNodeMgr(tb, conf, opt.WithRepoStore)
}
if opt.WithAssignmentStore == nil {
opt.WithAssignmentStore = NewDisabledAssignmentStore(conf.StorageNames())
}
if opt.WithRouter == nil {
opt.WithRouter = NewNodeManagerRouter(opt.WithNodeMgr, opt.WithRepoStore)
}
if opt.WithChecks == nil {
opt.WithChecks = service.Checks()
}
coordinator := NewCoordinator(
opt.WithLogger,
opt.WithQueue,
opt.WithRepoStore,
nil,
opt.WithRouter,
opt.WithTxMgr,
conf,
opt.WithAnnotations,
)
// TODO: run a replmgr for EVERY virtual storage
replmgr := NewReplMgr(
opt.WithLogger,
conf.StorageNames(),
opt.WithQueue,
opt.WithRepoStore,
opt.WithNodeMgr,
NodeSetFromNodeManager(opt.WithNodeMgr),
)
// We set up the structerr interceptors so that any error metadata that gets set via
// `structerr.WithMetadata()` is not only logged, but also present in the error details.
serverOpts := []ServerOption{
WithUnaryInterceptor(testserver.StructErrUnaryInterceptor),
WithStreamInterceptor(testserver.StructErrStreamInterceptor),
}
prf := NewGRPCServer(&Dependencies{
Config: conf,
Logger: opt.WithLogger,
Coordinator: coordinator,
Director: coordinator.StreamDirector,
NodeMgr: nil,
TxMgr: opt.WithTxMgr,
Queue: nil,
RepositoryStore: opt.WithRepoStore,
AssignmentStore: opt.WithAssignmentStore,
Router: opt.WithRouter,
Registry: protoregistry.GitalyProtoPreregistered,
Conns: opt.WithConnections,
PrimaryGetter: opt.WithPrimaryGetter,
Checks: opt.WithChecks,
}, nil, serverOpts...)
listener, port := listenAvailPort(tb)
errQ := make(chan error)
go func() {
errQ <- prf.Serve(listener)
close(errQ)
}()
replMgrDone := startProcessBacklog(ctx, replmgr)
// dial client to praefect
cc := dialLocalPort(tb, ctx, port, false)
cleanup := func() {
cc.Close()
for _, cu := range cleanups {
cu()
}
prf.Stop()
cancel()
<-replMgrDone
// A test that stops the server right after starting it may stop it before the goroutine
// above calls Serve. Serve then closes the listener and returns ErrServerStopped, the
// outcome of the stop, as for newMockDownstream.
if err := <-errQ; !errors.Is(err, grpc.ErrServerStopped) {
require.NoError(tb, err)
}
require.NoError(tb, db.Close())
}
return cc, prf, cleanup
}