Git designed for AI agents.
package praefect
import (
"context"
"errors"
"fmt"
"math/rand"
"strconv"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/feder-cr/git-ai/internal/cache"
"github.com/feder-cr/git-ai/internal/datastructure"
"github.com/feder-cr/git-ai/internal/featureflag"
"github.com/feder-cr/git-ai/internal/git/gittest"
gconfig "github.com/feder-cr/git-ai/internal/gitaly/config"
"github.com/feder-cr/git-ai/internal/gitaly/service"
"github.com/feder-cr/git-ai/internal/gitaly/service/repository"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/grpc/client"
gitaly_metadata "github.com/feder-cr/git-ai/internal/grpc/metadata"
"github.com/feder-cr/git-ai/internal/grpc/protoregistry"
"github.com/feder-cr/git-ai/internal/praefect/config"
"github.com/feder-cr/git-ai/internal/praefect/datastore"
"github.com/feder-cr/git-ai/internal/praefect/nodes"
"github.com/feder-cr/git-ai/internal/praefect/transactions"
"github.com/feder-cr/git-ai/internal/structerr"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/feder-cr/git-ai/internal/testhelper/promtest"
"github.com/feder-cr/git-ai/internal/testhelper/testcfg"
"github.com/feder-cr/git-ai/internal/testhelper/testdb"
"github.com/feder-cr/git-ai/internal/testhelper/testserver"
"github.com/feder-cr/git-ai/internal/transaction/txinfo"
"github.com/feder-cr/git-ai/internal/transaction/voting"
"github.com/feder-cr/git-ai/proto/go/gitalypb"
"github.com/feder-cr/git-ai/proxy"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/labkit/correlation"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
func TestStreamDirectorReadOnlyEnforcement(t *testing.T) {
t.Parallel()
db := testdb.New(t)
for _, tc := range []struct {
desc string
readOnly bool
}{
{desc: "writable", readOnly: false},
{desc: "read-only", readOnly: true},
} {
t.Run(tc.desc, func(t *testing.T) {
db.TruncateAll(t)
const (
virtualStorage = "test-virtual-storage"
relativePath = "test-repository"
storage = "test-storage"
)
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: virtualStorage,
Nodes: []*config.Node{
{
Address: "tcp://gitaly-primary.example.com",
Storage: storage,
},
},
},
},
}
ctx := testhelper.Context(t)
rs := datastore.MockRepositoryStore{
GetConsistentStoragesFunc: func(context.Context, string, string) (string, *datastructure.Set[string], error) {
if tc.readOnly {
return "", datastructure.SetFromValues(storage + "-other"), nil
}
return "", datastructure.NewSet[string](), nil
},
}
logger := testhelper.NewLogger(t)
repoWriteLockMgr := datastore.NewRepoReferenceWriteLockManager(ctx, db, testdb.GetConfig(t, db.Name), logger)
coordinator := NewCoordinator(
logger,
datastore.NewPostgresReplicationEventQueue(db),
rs,
nil,
NewNodeManagerRouter(&nodes.MockManager{GetShardFunc: func(vs string) (nodes.Shard, error) {
require.Equal(t, virtualStorage, vs)
return nodes.Shard{
Primary: &nodes.MockNode{GetStorageMethod: func() string {
return storage
}},
}, nil
}}, rs),
transactions.NewManager(conf, logger, repoWriteLockMgr),
conf,
protoregistry.GitalyProtoPreregistered,
)
frame, err := proto.Marshal(&gitalypb.DeleteRefsRequest{
Repository: &gitalypb.Repository{
StorageName: virtualStorage,
RelativePath: relativePath,
},
Refs: [][]byte{
[]byte("refs/heads/does-not-exist"),
},
})
require.NoError(t, err)
_, err = coordinator.StreamDirector(ctx, "/gitaly.RefService/DeleteRefs", &mockPeeker{frame: frame})
if tc.readOnly {
testhelper.RequireGrpcError(t, ErrRepositoryReadOnly, err)
testhelper.RequireGrpcCode(t, err, codes.FailedPrecondition)
} else {
require.NoError(t, err)
}
})
}
}
func TestStreamDirectorMutator(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
ctx = correlation.ContextWithCorrelation(ctx, "my-correlation-id")
primarySocket := testhelper.GetTemporaryGitalySocketFileName(t)
testhelper.NewServerWithHealth(t, primarySocket)
primaryNode := &config.Node{Address: "unix://" + primarySocket, Storage: "praefect-internal-1"}
secondarySocket := testhelper.GetTemporaryGitalySocketFileName(t)
testhelper.NewServerWithHealth(t, secondarySocket)
secondaryNode := &config.Node{Address: "unix://" + secondarySocket, Storage: "praefect-internal-2"}
unrelatedSocket := testhelper.GetTemporaryGitalySocketFileName(t)
testhelper.NewServerWithHealth(t, unrelatedSocket)
unrelatedNode := &config.Node{Address: "unix://" + unrelatedSocket, Storage: "praefect-unrelated"}
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{primaryNode, secondaryNode},
},
{
Name: "unrelated",
Nodes: []*config.Node{unrelatedNode},
},
},
}
db := testdb.New(t)
logger := testhelper.SharedLogger(t)
repoWriteLockMgr := datastore.NewRepoReferenceWriteLockManager(ctx, db, testdb.GetConfig(t, db.Name), logger)
txMgr := transactions.NewManager(conf, logger, repoWriteLockMgr)
nodeSet, err := DialNodes(ctx, conf.VirtualStorages, protoregistry.GitalyProtoPreregistered, nil, nil, nil, logger)
require.NoError(t, err)
defer nodeSet.Close()
type setupData struct {
method string
request proto.Message
expectedErr error
expectedRewrittenRequest proto.Message
expectedEvent datastore.ReplicationEvent
}
var currentRepoID int64
createRepo := func(t *testing.T, rs datastore.RepositoryStore, storage, relativePath, rewrittenPath string) int64 {
t.Helper()
repoID := atomic.AddInt64(¤tRepoID, 1)
require.NoError(t, rs.CreateRepository(ctx, repoID, storage, relativePath, rewrittenPath, primaryNode.Storage, []string{secondaryNode.Storage}, nil, true, true))
return repoID
}
for _, tc := range []struct {
desc string
setup func(t *testing.T, rs datastore.RepositoryStore) setupData
}{
{
desc: "target repository",
setup: func(t *testing.T, rs datastore.RepositoryStore) setupData {
relativePath := gittest.NewRepositoryName(t)
targetRepo := &gitalypb.Repository{
StorageName: "praefect",
RelativePath: relativePath,
}
repoID := createRepo(t, rs, "praefect", relativePath, relativePath)
return setupData{
method: "/gitaly.OperationService/UserCreateTag",
request: &gitalypb.UserCreateTagRequest{
Repository: targetRepo,
},
expectedRewrittenRequest: &gitalypb.UserCreateTagRequest{
Repository: &gitalypb.Repository{
StorageName: "praefect-internal-1",
RelativePath: relativePath,
},
},
expectedEvent: datastore.ReplicationEvent{
State: datastore.JobStateInProgress,
Attempt: 2,
LockID: fmt.Sprintf("praefect|praefect-internal-2|%s", relativePath),
Job: datastore.ReplicationJob{
RepositoryID: repoID,
Change: datastore.UpdateRepo,
VirtualStorage: conf.VirtualStorages[0].Name,
RelativePath: targetRepo.GetRelativePath(),
TargetNodeStorage: secondaryNode.Storage,
SourceNodeStorage: primaryNode.Storage,
},
Meta: datastore.Params{datastore.CorrelationIDKey: "my-correlation-id"},
},
}
},
},
{
desc: "target and additional repository",
setup: func(t *testing.T, rs datastore.RepositoryStore) setupData {
targetRepo := &gitalypb.Repository{
StorageName: "praefect",
RelativePath: gittest.NewRepositoryName(t),
}
additionalRepo := &gitalypb.Repository{
StorageName: "praefect",
RelativePath: gittest.NewRepositoryName(t),
}
targetRepoID := createRepo(t, rs, "praefect", targetRepo.GetRelativePath(), "rewritten-target")
createRepo(t, rs, "praefect", additionalRepo.GetRelativePath(), "rewritten-additional")
return setupData{
method: "/gitaly.ObjectPoolService/FetchIntoObjectPool",
request: &gitalypb.FetchIntoObjectPoolRequest{
Origin: additionalRepo,
ObjectPool: &gitalypb.ObjectPool{
Repository: targetRepo,
},
},
expectedRewrittenRequest: &gitalypb.FetchIntoObjectPoolRequest{
Origin: &gitalypb.Repository{
StorageName: "praefect-internal-1",
RelativePath: "rewritten-additional",
},
ObjectPool: &gitalypb.ObjectPool{
Repository: &gitalypb.Repository{
StorageName: "praefect-internal-1",
RelativePath: "rewritten-target",
},
},
},
expectedEvent: datastore.ReplicationEvent{
State: datastore.JobStateInProgress,
Attempt: 2,
LockID: fmt.Sprintf("praefect|praefect-internal-2|%s", targetRepo.GetRelativePath()),
Job: datastore.ReplicationJob{
RepositoryID: targetRepoID,
Change: datastore.UpdateRepo,
VirtualStorage: conf.VirtualStorages[0].Name,
RelativePath: targetRepo.GetRelativePath(),
TargetNodeStorage: secondaryNode.Storage,
SourceNodeStorage: primaryNode.Storage,
},
Meta: datastore.Params{datastore.CorrelationIDKey: "my-correlation-id"},
},
}
},
},
{
desc: "target repository not found",
setup: func(t *testing.T, rs datastore.RepositoryStore) setupData {
// We do not create the repository, so we expect this request to
// fail.
targetRepo := &gitalypb.Repository{
StorageName: "praefect",
RelativePath: gittest.NewRepositoryName(t),
}
return setupData{
method: "/gitaly.OperationService/UserCreateTag",
request: &gitalypb.UserCreateTagRequest{
Repository: targetRepo,
},
expectedErr: storage.NewRepositoryNotFoundError(targetRepo.GetStorageName(), targetRepo.GetRelativePath()),
}
},
},
{
desc: "additional repository not found",
setup: func(t *testing.T, rs datastore.RepositoryStore) setupData {
targetRepo := &gitalypb.Repository{
StorageName: "praefect",
RelativePath: gittest.NewRepositoryName(t),
}
additionalRepo := &gitalypb.Repository{
StorageName: "praefect",
RelativePath: gittest.NewRepositoryName(t),
}
// We create the target repository, but don't create the additional
// one.
createRepo(t, rs, "praefect", targetRepo.GetRelativePath(), "rewritten-target")
return setupData{
method: "/gitaly.ObjectPoolService/FetchIntoObjectPool",
request: &gitalypb.FetchIntoObjectPoolRequest{
Origin: additionalRepo,
ObjectPool: &gitalypb.ObjectPool{
Repository: targetRepo,
},
},
expectedErr: structerr.NewNotFound("%w", fmt.Errorf("mutator call: route repository mutator: %w",
fmt.Errorf("resolve additional replica path: %w",
additionalRepositoryNotFoundError{
storageName: additionalRepo.GetStorageName(),
relativePath: additionalRepo.GetRelativePath(),
},
),
)).WithMetadataItems(
structerr.MetadataItem{Key: "storage_name", Value: additionalRepo.GetStorageName()},
structerr.MetadataItem{Key: "relative_path", Value: additionalRepo.GetRelativePath()},
),
}
},
},
{
desc: "target and additional repository on different storages",
setup: func(t *testing.T, rs datastore.RepositoryStore) setupData {
targetRepo := &gitalypb.Repository{
StorageName: "praefect",
RelativePath: gittest.NewRepositoryName(t),
}
additionalRepo := &gitalypb.Repository{
StorageName: "unrelated",
RelativePath: gittest.NewRepositoryName(t),
}
createRepo(t, rs, "praefect", targetRepo.GetRelativePath(), "rewritten-target")
createRepo(t, rs, "unrelated", additionalRepo.GetRelativePath(), "rewritten-additional")
return setupData{
method: "/gitaly.ObjectPoolService/FetchIntoObjectPool",
request: &gitalypb.FetchIntoObjectPoolRequest{
Origin: additionalRepo,
ObjectPool: &gitalypb.ObjectPool{
Repository: targetRepo,
},
},
expectedErr: structerr.NewInvalidArgument("resolving additional repository on different storage than target repository is not supported"),
}
},
},
} {
t.Run(tc.desc, func(t *testing.T) {
tx := db.Begin(t)
defer tx.Rollback(t)
rs := datastore.NewPostgresRepositoryStore(tx, conf.StorageNames())
logger := testhelper.NewLogger(t)
testdb.SetHealthyNodes(t, ctx, tx, map[string]map[string][]string{"praefect": conf.StorageNames()})
queueInterceptor := datastore.NewReplicationEventQueueInterceptor(datastore.NewPostgresReplicationEventQueue(tx))
queueInterceptor.OnEnqueue(func(ctx context.Context, event datastore.ReplicationEvent, queue datastore.ReplicationEventQueue) (datastore.ReplicationEvent, error) {
assert.True(t, len(queueInterceptor.GetEnqueued()) < 2, "expected only one event to be created")
return queue.Enqueue(ctx, event)
})
coordinator := NewCoordinator(
logger,
queueInterceptor,
rs,
nil,
NewPerRepositoryRouter(
nodeSet.Connections(),
nodes.NewPerRepositoryElector(logger, tx),
StaticHealthChecker(conf.StorageNames()),
NewLockedRandom(rand.New(rand.NewSource(0))),
rs,
datastore.NewAssignmentStore(tx, conf.StorageNames()),
rs,
nil,
),
txMgr,
conf,
protoregistry.GitalyProtoPreregistered,
)
setup := tc.setup(t, rs)
marshalledRequest, err := proto.Marshal(setup.request)
require.NoError(t, err)
streamParams, err := coordinator.StreamDirector(ctx, setup.method, &mockPeeker{marshalledRequest})
require.Equal(t, setup.expectedErr, err)
if setup.expectedErr != nil {
return
}
// Assert that the stream parameters look as expecected. First, we verify
// that the request gets routed to the correct primary node.
require.Equal(t, primaryNode.Address, streamParams.Primary().Conn.Target())
// Second, we verify that the request was rewritten so that its storage
// matches the primary node's storage.
mi, err := coordinator.registry.LookupMethod(setup.method)
require.NoError(t, err)
rewrittenRequest, err := mi.UnmarshalRequestProto(streamParams.Primary().Msg)
require.NoError(t, err)
testhelper.ProtoEqual(t, setup.expectedRewrittenRequest, rewrittenRequest)
// Finalize the request and then wait for the resulting replication event to
// be queued.
require.NoError(t, streamParams.RequestFinalizer())
require.NoError(t, queueInterceptor.Wait(time.Minute, func(i *datastore.ReplicationEventQueueInterceptor) bool {
return len(i.GetEnqueuedResult()) == 1
}))
// Assert that the replication event queue contains the expected event now.
events, err := queueInterceptor.Dequeue(ctx, "praefect", "praefect-internal-2", 10)
require.NoError(t, err)
require.Len(t, events, 1)
// Unset a bunch of data that is indeterministic.
events[0].ID = 0
events[0].CreatedAt = time.Time{}
events[0].UpdatedAt = nil
require.Equal(t, setup.expectedEvent, events[0])
})
}
}
func TestStreamDirectorMutator_StopTransaction(t *testing.T) {
t.Parallel()
socket := testhelper.GetTemporaryGitalySocketFileName(t)
testhelper.NewServerWithHealth(t, socket)
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{
{Address: "unix://" + socket, Storage: "primary"},
{Address: "unix://" + socket, Storage: "secondary"},
},
},
},
}
repo := gitalypb.Repository{
StorageName: "praefect",
RelativePath: "/path/to/hashed/storage",
}
nodeMgr, err := nodes.NewManager(testhelper.SharedLogger(t), conf, nil, nil, promtest.NewMockHistogramVec(), protoregistry.GitalyProtoPreregistered, nil, nil, nil)
require.NoError(t, err)
nodeMgr.Start(0, time.Hour)
defer nodeMgr.Stop()
ctx := testhelper.Context(t)
shard, err := nodeMgr.GetShard(ctx, conf.VirtualStorages[0].Name)
require.NoError(t, err)
for _, name := range []string{"primary", "secondary"} {
node, err := shard.GetNode(name)
require.NoError(t, err)
waitNodeToChangeHealthStatus(t, ctx, node, true)
}
rs := datastore.MockRepositoryStore{
GetConsistentStoragesFunc: func(ctx context.Context, virtualStorage, relativePath string) (string, *datastructure.Set[string], error) {
return relativePath, datastructure.SetFromValues("primary", "secondary"), nil
},
}
logger := testhelper.NewLogger(t)
txMgr := transactions.NewManager(conf, logger, &datastore.NoopWriteLockManager{})
coordinator := NewCoordinator(
logger,
datastore.NewPostgresReplicationEventQueue(testdb.New(t)),
rs,
nil,
NewNodeManagerRouter(nodeMgr, rs),
txMgr,
conf,
protoregistry.GitalyProtoPreregistered,
)
fullMethod := "/gitaly.SmartHTTPService/PostReceivePack"
frame, err := proto.Marshal(&gitalypb.PostReceivePackRequest{
Repository: &repo,
})
require.NoError(t, err)
peeker := &mockPeeker{frame}
streamParams, err := coordinator.StreamDirector(correlation.ContextWithCorrelation(ctx, "my-correlation-id"), fullMethod, peeker)
require.NoError(t, err)
txCtx := peer.NewContext(streamParams.Primary().Ctx, &peer.Peer{})
transaction, err := txinfo.TransactionFromContext(txCtx)
require.NoError(t, err)
var wg sync.WaitGroup
var syncWG sync.WaitGroup
wg.Add(2)
syncWG.Add(2)
go func() {
defer wg.Done()
vote := voting.VoteFromData([]byte("vote"))
err := txMgr.VoteTransaction(ctx, transaction.ID, repo.GetStorageName(), repo.GetRelativePath(), "primary",
gitalypb.VoteTransactionRequest_PREPARING_PHASE, vote)
require.NoError(t, err)
// Assure that at least one vote was agreed on.
syncWG.Done()
syncWG.Wait()
require.NoError(t, txMgr.StopTransaction(ctx, transaction.ID))
}()
go func() {
defer wg.Done()
vote := voting.VoteFromData([]byte("vote"))
err := txMgr.VoteTransaction(ctx, transaction.ID, repo.GetStorageName(), repo.GetRelativePath(), "secondary",
gitalypb.VoteTransactionRequest_PREPARING_PHASE, vote)
require.NoError(t, err)
// Assure that at least one vote was agreed on.
syncWG.Done()
syncWG.Wait()
err = txMgr.VoteTransaction(ctx, transaction.ID, repo.GetStorageName(), repo.GetRelativePath(), "secondary",
gitalypb.VoteTransactionRequest_PREPARING_PHASE, vote)
assert.True(t, errors.Is(err, transactions.ErrTransactionStopped))
}()
wg.Wait()
err = streamParams.RequestFinalizer()
require.NoError(t, err)
}
func TestStreamDirectorMutator_SecondaryErrorHandling(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
socket := testhelper.GetTemporaryGitalySocketFileName(t)
testhelper.NewServerWithHealth(t, socket)
primaryNode := &config.Node{Address: "unix://" + socket, Storage: "praefect-internal-1"}
secondaryNode1 := &config.Node{Address: "unix://" + socket, Storage: "praefect-internal-2"}
secondaryNode2 := &config.Node{Address: "unix://" + socket, Storage: "praefect-internal-3"}
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{primaryNode, secondaryNode1, secondaryNode2},
},
},
}
repo := gitalypb.Repository{
StorageName: "praefect",
RelativePath: "/path/to/hashed/storage",
}
nodeMgr, err := nodes.NewManager(testhelper.SharedLogger(t), conf, nil, nil, promtest.NewMockHistogramVec(), protoregistry.GitalyProtoPreregistered, nil, nil, nil)
require.NoError(t, err)
nodeMgr.Start(0, time.Hour)
defer nodeMgr.Stop()
shard, err := nodeMgr.GetShard(ctx, conf.VirtualStorages[0].Name)
require.NoError(t, err)
for _, name := range []string{"praefect-internal-1", "praefect-internal-2", "praefect-internal-3"} {
node, err := shard.GetNode(name)
require.NoError(t, err)
waitNodeToChangeHealthStatus(t, ctx, node, true)
}
rs := datastore.MockRepositoryStore{
GetConsistentStoragesFunc: func(ctx context.Context, virtualStorage, relativePath string) (string, *datastructure.Set[string], error) {
return relativePath, datastructure.SetFromValues("praefect-internal-1", "praefect-internal-2", "praefect-internal-3"), nil
},
}
logger := testhelper.NewLogger(t)
txMgr := transactions.NewManager(conf, logger, &datastore.NoopWriteLockManager{})
coordinator := NewCoordinator(
logger,
datastore.NewPostgresReplicationEventQueue(testdb.New(t)),
rs,
nil,
NewNodeManagerRouter(nodeMgr, rs),
txMgr,
conf,
protoregistry.GitalyProtoPreregistered,
)
fullMethod := "/gitaly.SmartHTTPService/PostReceivePack"
frame, err := proto.Marshal(&gitalypb.PostReceivePackRequest{
Repository: &repo,
})
require.NoError(t, err)
peeker := &mockPeeker{frame}
streamParams, err := coordinator.StreamDirector(correlation.ContextWithCorrelation(ctx, "my-correlation-id"), fullMethod, peeker)
require.NoError(t, err)
txCtx := peer.NewContext(streamParams.Primary().Ctx, &peer.Peer{})
transaction, err := txinfo.TransactionFromContext(txCtx)
require.NoError(t, err)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
vote := voting.VoteFromData([]byte("vote"))
err := txMgr.VoteTransaction(ctx, transaction.ID, repo.GetStorageName(), repo.GetRelativePath(), "praefect-internal-1",
gitalypb.VoteTransactionRequest_PREPARING_PHASE, vote)
require.ErrorIs(t, err, transactions.ErrTransactionFailed)
}()
for _, secondary := range streamParams.Secondaries() {
wg.Add(1)
go func() {
err := secondary.ErrHandler(errors.New("node RPC failure"))
require.NoError(t, err)
wg.Done()
}()
}
wg.Wait()
err = streamParams.RequestFinalizer()
require.NoError(t, err)
}
func TestStreamDirectorMutator_ReplicateRepository(t *testing.T) {
for _, tc := range []struct {
Name string
Request *gitalypb.ReplicateRepositoryRequest
PartitioningHint string
}{
{
Name: "invoked without a partitioning hint",
Request: &gitalypb.ReplicateRepositoryRequest{
Repository: &gitalypb.Repository{
StorageName: "praefect-2",
RelativePath: "/path/to/hashed/storage",
},
Source: &gitalypb.Repository{
StorageName: "praefect-1",
RelativePath: "/path/to/hashed/storage",
},
},
},
{
Name: "invoked with a partitioning hint",
Request: &gitalypb.ReplicateRepositoryRequest{
Repository: &gitalypb.Repository{
StorageName: "praefect-2",
RelativePath: "/path/to/hashed/storage",
},
Source: &gitalypb.Repository{
StorageName: "praefect-1",
RelativePath: "/path/to/hashed/storage",
},
},
PartitioningHint: "/path/to/hashed/storage/hint",
},
} {
t.Run(tc.Name, func(t *testing.T) {
ctx := testhelper.Context(t)
socket := testhelper.GetTemporaryGitalySocketFileName(t)
testhelper.NewServerWithHealth(t, socket)
// Setup config with two virtual storages.
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect-1",
Nodes: []*config.Node{{Address: "unix://" + socket, Storage: "praefect-internal-1"}},
},
{
Name: "praefect-2",
Nodes: []*config.Node{{Address: "unix://" + socket, Storage: "praefect-internal-2"}},
},
},
}
nodeMgr, err := nodes.NewManager(testhelper.SharedLogger(t), conf, nil, nil, promtest.NewMockHistogramVec(), protoregistry.GitalyProtoPreregistered, nil, nil, nil)
require.NoError(t, err)
nodeMgr.Start(0, time.Hour)
defer nodeMgr.Stop()
incrementGenerationInvoked := false
rs := datastore.MockRepositoryStore{
GetConsistentStoragesFunc: func(ctx context.Context, virtualStorage, relativePath string) (string, *datastructure.Set[string], error) {
return relativePath, datastructure.SetFromValues("praefect-internal-2"), nil
},
CreateRepositoryFunc: func(ctx context.Context, repositoryID int64, virtualStorage, relativePath, replicaPath, primary string, updatedSecondaries, outdatedSecondaries []string, storePrimary, storeAssignments bool) error {
require.Fail(t, "CreateRepository should not be called")
return nil
},
IncrementGenerationFunc: func(ctx context.Context, repositoryID int64, primary string, secondaries []string) error {
incrementGenerationInvoked = true
return nil
},
}
router := mockRouter{
// Simulate scenario where target repository already exists and error is returned.
routeRepositoryCreation: func(ctx context.Context, virtualStorage, relativePath, additionalRepoRelativePath string) (RepositoryMutatorRoute, error) {
return RepositoryMutatorRoute{}, fmt.Errorf("reserve repository id: %w", datastore.ErrRepositoryAlreadyExists)
},
// Pass through normally to handle route creation.
routeRepositoryMutator: func(ctx context.Context, virtualStorage, relativePath, additionalRepoRelativePath string) (RepositoryMutatorRoute, error) {
return NewNodeManagerRouter(nodeMgr, rs).RouteRepositoryMutator(ctx, virtualStorage, relativePath, additionalRepoRelativePath)
},
}
logger := testhelper.NewLogger(t)
coordinator := NewCoordinator(
logger,
&datastore.MockReplicationEventQueue{},
rs,
nil,
router,
transactions.NewManager(conf, logger, &datastore.NoopWriteLockManager{}),
conf,
protoregistry.GitalyProtoPreregistered,
)
fullMethod := "/gitaly.RepositoryService/ReplicateRepository"
frame, err := proto.Marshal(tc.Request)
require.NoError(t, err)
peeker := &mockPeeker{frame}
ctx = correlation.ContextWithCorrelation(ctx, "my-correlation-id")
if tc.PartitioningHint != "" {
ctx = storage.ContextWithPartitioningHint(ctx, tc.PartitioningHint)
}
// Validate that stream parameters can be constructed successfully for
// `ReplicateRepository` when the target repository already exists.
streamParams, err := coordinator.StreamDirector(ctx, fullMethod, peeker)
require.NoError(t, err)
// Validate the partitioning hint was passed through.
hint, err := storage.ExtractPartitioningHint(ctx)
require.NoError(t, err)
require.Equal(t, tc.PartitioningHint, hint)
// Validate that `CreateRepository()` is not invoked and `IncrementGeneration()`
// is when target repository already exists.
err = streamParams.RequestFinalizer()
require.NoError(t, err)
require.True(t, incrementGenerationInvoked)
})
}
}
func TestStreamDirector_maintenance(t *testing.T) {
t.Parallel()
node1 := &config.Node{
Address: "unix://" + testhelper.GetTemporaryGitalySocketFileName(t),
Storage: "praefect-internal-1",
}
node2 := &config.Node{
Address: "unix://" + testhelper.GetTemporaryGitalySocketFileName(t),
Storage: "praefect-internal-2",
}
cfg := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{node1, node2},
},
},
}
db := testdb.New(t)
logger := testhelper.NewLogger(t)
repo := gitalypb.Repository{
StorageName: "praefect",
RelativePath: "/path/to/hashed/storage",
}
ctx := testhelper.Context(t)
nodeSet, err := DialNodes(ctx, cfg.VirtualStorages, protoregistry.GitalyProtoPreregistered, nil, nil, nil, logger)
require.NoError(t, err)
defer nodeSet.Close()
tx := db.Begin(t)
defer tx.Rollback(t)
rs := datastore.NewPostgresRepositoryStore(tx, cfg.StorageNames())
require.NoError(t, rs.CreateRepository(ctx, 1, repo.GetStorageName(), repo.GetRelativePath(),
repo.GetRelativePath(), node1.Storage, []string{node2.Storage}, nil, true, true))
testdb.SetHealthyNodes(t, ctx, tx, map[string]map[string][]string{"praefect": cfg.StorageNames()})
queueInterceptor := datastore.NewReplicationEventQueueInterceptor(datastore.NewPostgresReplicationEventQueue(tx))
coordinator := NewCoordinator(
logger,
queueInterceptor,
rs,
nil,
NewPerRepositoryRouter(
nodeSet.Connections(),
nodes.NewPerRepositoryElector(logger, tx),
StaticHealthChecker(cfg.StorageNames()),
NewLockedRandom(rand.New(rand.NewSource(0))),
rs,
datastore.NewAssignmentStore(tx, cfg.StorageNames()),
rs,
nil,
),
nil,
cfg,
protoregistry.GitalyProtoPreregistered,
)
message, err := proto.Marshal(&gitalypb.OptimizeRepositoryRequest{
Repository: &repo,
})
require.NoError(t, err)
methodInfo, err := protoregistry.GitalyProtoPreregistered.LookupMethod("/gitaly.RepositoryService/OptimizeRepository")
require.NoError(t, err)
for _, tc := range []struct {
desc string
primaryErr error
secondaryErr error
expectedErr error
}{
{
desc: "successful",
},
{
desc: "primary returns an error",
primaryErr: structerr.NewNotFound("primary error"),
expectedErr: structerr.NewNotFound("primary error"),
},
{
desc: "secondary returns an error",
secondaryErr: structerr.NewNotFound("secondary error"),
expectedErr: structerr.NewNotFound("secondary error"),
},
{
desc: "primary error preferred",
primaryErr: structerr.NewNotFound("primary error"),
secondaryErr: structerr.NewNotFound("secondary error"),
expectedErr: structerr.NewNotFound("primary error"),
},
} {
t.Run(tc.desc, func(t *testing.T) {
// Behaviour between the new and old routing are sufficiently different that
// it doesn't make sense to try and cram both tests in here.
queueInterceptor.OnEnqueue(func(context.Context, datastore.ReplicationEvent, datastore.ReplicationEventQueue) (datastore.ReplicationEvent, error) {
require.FailNow(t, "no replication jobs should have been created")
return datastore.ReplicationEvent{}, fmt.Errorf("unexpected call")
})
streamParams, err := coordinator.StreamDirector(ctx, methodInfo.FullMethodName(), &mockPeeker{message})
require.NoError(t, err)
var targetNodes []string
destinationByAddress := make(map[string]proxy.Destination)
for _, node := range append(streamParams.Secondaries(), streamParams.Primary()) {
targetNodes = append(targetNodes, node.Conn.Target())
destinationByAddress[node.Conn.Target()] = node
}
// Assert that both nodes are part of the stream parameters. Because the
// order is not deterministic (we randomly shuffle primary and secondary
// nodes) we only assert that elements match, not that any of both nodes has
// a specific role.
require.ElementsMatch(t, []string{
node1.Address,
node2.Address,
}, targetNodes)
// Assert that the target repositories were rewritten as expected for all of
// the nodes.
for _, nodeCfg := range []*config.Node{node1, node2} {
destination, ok := destinationByAddress[nodeCfg.Address]
require.True(t, ok)
request, err := methodInfo.UnmarshalRequestProto(destination.Msg)
require.NoError(t, err)
rewrittenTargetRepo, err := methodInfo.TargetRepo(request)
require.NoError(t, err)
require.Equal(t, nodeCfg.Storage, rewrittenTargetRepo.GetStorageName(), "stream director should have rewritten the storage name")
}
// We now inject errors into the streams by manually executing the error
// handlers. This simulates that the RPCs have been proxied and may or may
// not have returned an error.
//
// Note that these functions should not return an error: this is a strict
// requirement because otherwise failures returned by the primary would
// abort the operation on secondaries.
require.Nil(t, streamParams.Primary().ErrHandler(tc.primaryErr))
require.Nil(t, streamParams.Secondaries()[0].ErrHandler(tc.secondaryErr))
// The request finalizer should then see the errors as injected above.
require.Equal(t, tc.expectedErr, streamParams.RequestFinalizer())
})
}
}
func TestStreamDirector_maintenanceRPCs(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
primaryStorage := "internal-gitaly-0"
primaryCfg := testcfg.Build(t, testcfg.WithStorages(primaryStorage))
primaryServer, primarySocketPath := runMockMaintenanceServer(t, primaryCfg)
primaryCfg.SocketPath = primarySocketPath
secondaryStorage := "internal-gitaly-1"
secondaryCfg := testcfg.Build(t, testcfg.WithStorages(secondaryStorage))
secondaryServer, secondarySocketPath := runMockMaintenanceServer(t, secondaryCfg)
secondaryCfg.SocketPath = secondarySocketPath
cc, _, cleanup := RunPraefectServer(t, ctx, config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "default",
Nodes: []*config.Node{
{
Storage: primaryStorage,
Address: primarySocketPath,
},
{
Storage: secondaryStorage,
Address: secondarySocketPath,
},
},
},
},
}, BuildOptions{})
defer cleanup()
repository := &gitalypb.Repository{
StorageName: "default",
RelativePath: gittest.NewRepositoryName(t),
}
primaryRepository := &gitalypb.Repository{
StorageName: primaryStorage,
RelativePath: repository.GetRelativePath(),
}
secondaryRepository := &gitalypb.Repository{
StorageName: secondaryStorage,
RelativePath: repository.GetRelativePath(),
}
repositoryClient := gitalypb.NewRepositoryServiceClient(cc)
gittest.CreateRepository(t, ctx, primaryCfg, gittest.CreateRepositoryConfig{
RelativePath: repository.GetRelativePath(),
})
gittest.CreateRepository(t, ctx, secondaryCfg, gittest.CreateRepositoryConfig{
RelativePath: repository.GetRelativePath(),
})
for _, tc := range []struct {
desc string
maintenanceFunc func(t *testing.T)
expectedPrimaryRequest proto.Message
expectedSecondaryRequest proto.Message
}{
{
desc: "OptimizeRepository",
maintenanceFunc: func(t *testing.T) {
_, err := repositoryClient.OptimizeRepository(ctx, &gitalypb.OptimizeRepositoryRequest{
Repository: repository,
})
require.NoError(t, err)
},
expectedPrimaryRequest: &gitalypb.OptimizeRepositoryRequest{
Repository: primaryRepository,
},
expectedSecondaryRequest: &gitalypb.OptimizeRepositoryRequest{
Repository: secondaryRepository,
},
},
{
desc: "PruneUnreachableObjects",
maintenanceFunc: func(t *testing.T) {
_, err := repositoryClient.PruneUnreachableObjects(ctx, &gitalypb.PruneUnreachableObjectsRequest{
Repository: repository,
})
require.NoError(t, err)
},
expectedPrimaryRequest: &gitalypb.PruneUnreachableObjectsRequest{
Repository: primaryRepository,
},
expectedSecondaryRequest: &gitalypb.PruneUnreachableObjectsRequest{
Repository: secondaryRepository,
},
},
} {
t.Run(tc.desc, func(t *testing.T) {
tc.maintenanceFunc(t)
testhelper.ProtoEqual(t, tc.expectedPrimaryRequest, <-primaryServer.requestCh)
testhelper.ProtoEqual(t, tc.expectedSecondaryRequest, <-secondaryServer.requestCh)
})
}
}
type mockMaintenanceServer struct {
requestCh chan proto.Message
gitalypb.RepositoryServiceServer
}
func runMockMaintenanceServer(t *testing.T, cfg gconfig.Cfg) (*mockMaintenanceServer, string) {
server := &mockMaintenanceServer{
requestCh: make(chan proto.Message, 1),
}
addr := testserver.RunGitalyServer(t, cfg, func(srv *grpc.Server, deps *service.Dependencies) {
server.RepositoryServiceServer = repository.NewServer(deps)
gitalypb.RegisterRepositoryServiceServer(srv, server)
}, testserver.WithDisablePraefect())
return server, addr
}
func (m *mockMaintenanceServer) OptimizeRepository(ctx context.Context, in *gitalypb.OptimizeRepositoryRequest) (*gitalypb.OptimizeRepositoryResponse, error) {
if tx := storage.ExtractTransaction(ctx); tx != nil {
in.Repository = tx.OriginalRepository(in.GetRepository())
}
m.requestCh <- in
return &gitalypb.OptimizeRepositoryResponse{}, nil
}
func (m *mockMaintenanceServer) PruneUnreachableObjects(ctx context.Context, in *gitalypb.PruneUnreachableObjectsRequest) (*gitalypb.PruneUnreachableObjectsResponse, error) {
if tx := storage.ExtractTransaction(ctx); tx != nil {
in.Repository = tx.OriginalRepository(in.GetRepository())
}
m.requestCh <- in
return &gitalypb.PruneUnreachableObjectsResponse{}, nil
}
type mockRouter struct {
Router
routeRepositoryAccessorFunc func(ctx context.Context, virtualStorage, relativePath string, forcePrimary bool) (RepositoryAccessorRoute, error)
routeRepositoryCreation func(ctx context.Context, virtualStorage, relativePath, additionalRepoRelativePath string) (RepositoryMutatorRoute, error)
routeRepositoryMutator func(ctx context.Context, virtualStorage, relativePath, additionalRepoRelativePath string) (RepositoryMutatorRoute, error)
}
func (m mockRouter) RouteRepositoryAccessor(ctx context.Context, virtualStorage, relativePath string, forcePrimary bool) (RepositoryAccessorRoute, error) {
return m.routeRepositoryAccessorFunc(ctx, virtualStorage, relativePath, forcePrimary)
}
func (m mockRouter) RouteRepositoryCreation(ctx context.Context, virtualStorage, relativePath, additionalRepoRelativePath string) (RepositoryMutatorRoute, error) {
return m.routeRepositoryCreation(ctx, virtualStorage, relativePath, additionalRepoRelativePath)
}
func (m mockRouter) RouteRepositoryMutator(ctx context.Context, virtualStorage, relativePath, additionalRepoRelativePath string) (RepositoryMutatorRoute, error) {
return m.routeRepositoryMutator(ctx, virtualStorage, relativePath, additionalRepoRelativePath)
}
func TestStreamDirectorAccessor(t *testing.T) {
t.Parallel()
gitalySocket := testhelper.GetTemporaryGitalySocketFileName(t)
testhelper.NewServerWithHealth(t, gitalySocket)
gitalyAddress := "unix://" + gitalySocket
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{
{
Address: gitalyAddress,
Storage: "praefect-internal-1",
},
},
},
},
}
queue := datastore.NewPostgresReplicationEventQueue(testdb.New(t))
targetRepo := gitalypb.Repository{
StorageName: "praefect",
RelativePath: "/path/to/hashed/storage",
}
ctx := testhelper.Context(t)
logger := testhelper.SharedLogger(t)
rs := datastore.MockRepositoryStore{}
nodeMgr, err := nodes.NewManager(logger, conf, nil, rs, promtest.NewMockHistogramVec(), protoregistry.GitalyProtoPreregistered, nil, nil, nil)
require.NoError(t, err)
nodeMgr.Start(0, time.Minute)
defer nodeMgr.Stop()
txMgr := transactions.NewManager(conf, logger, &datastore.NoopWriteLockManager{})
for _, tc := range []struct {
desc string
router Router
error error
}{
{
desc: "success",
router: NewNodeManagerRouter(nodeMgr, rs),
},
{
desc: "repository not found",
router: mockRouter{
routeRepositoryAccessorFunc: func(_ context.Context, virtualStorage, relativePath string, _ bool) (RepositoryAccessorRoute, error) {
return RepositoryAccessorRoute{}, datastore.ErrRepositoryNotFound
},
},
error: structerr.NewNotFound("%w", datastore.ErrRepositoryNotFound),
},
} {
t.Run(tc.desc, func(t *testing.T) {
coordinator := NewCoordinator(
testhelper.NewLogger(t),
queue,
rs,
nil,
tc.router,
txMgr,
conf,
protoregistry.GitalyProtoPreregistered,
)
frame, err := proto.Marshal(&gitalypb.FindAllBranchesRequest{Repository: &targetRepo})
require.NoError(t, err)
fullMethod := "/gitaly.RefService/FindAllBranches"
peeker := &mockPeeker{frame: frame}
streamParams, err := coordinator.StreamDirector(correlation.ContextWithCorrelation(ctx, "my-correlation-id"), fullMethod, peeker)
if tc.error != nil {
testhelper.RequireGrpcError(t, tc.error, err)
return
}
require.NoError(t, err)
require.Equal(t, gitalyAddress, streamParams.Primary().Conn.Target())
mi, err := coordinator.registry.LookupMethod(fullMethod)
require.NoError(t, err)
require.Equal(t, protoregistry.ScopeRepository, mi.Scope, "method must be repository scoped")
require.Equal(t, protoregistry.OpAccessor, mi.Operation, "method must be an accessor")
m, err := mi.UnmarshalRequestProto(streamParams.Primary().Msg)
require.NoError(t, err)
rewrittenTargetRepo, err := mi.TargetRepo(m)
require.NoError(t, err)
require.Equal(t, "praefect-internal-1", rewrittenTargetRepo.GetStorageName(), "stream director should have rewritten the storage name")
})
}
}
func TestCoordinatorStreamDirector_distributesReads(t *testing.T) {
t.Parallel()
gitalySocket0, gitalySocket1 := testhelper.GetTemporaryGitalySocketFileName(t), testhelper.GetTemporaryGitalySocketFileName(t)
primaryHealthSrv := testhelper.NewServerWithHealth(t, gitalySocket0)
healthSrv := testhelper.NewServerWithHealth(t, gitalySocket1)
primaryNodeConf := config.Node{
Address: "unix://" + gitalySocket0,
Storage: "gitaly-1",
}
secondaryNodeConf := config.Node{
Address: "unix://" + gitalySocket1,
Storage: "gitaly-2",
}
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{&primaryNodeConf, &secondaryNodeConf},
},
},
Failover: config.Failover{
Enabled: true,
ElectionStrategy: "local",
},
}
queue := datastore.NewPostgresReplicationEventQueue(testdb.New(t))
targetRepo := gitalypb.Repository{
StorageName: "praefect",
RelativePath: "/path/to/hashed/storage",
}
ctx := testhelper.Context(t)
logger := testhelper.SharedLogger(t)
repoStore := datastore.MockRepositoryStore{
GetConsistentStoragesFunc: func(ctx context.Context, virtualStorage, relativePath string) (string, *datastructure.Set[string], error) {
return relativePath, datastructure.SetFromValues(primaryNodeConf.Storage, secondaryNodeConf.Storage), nil
},
}
nodeMgr, err := nodes.NewManager(logger, conf, nil, repoStore, promtest.NewMockHistogramVec(), protoregistry.GitalyProtoPreregistered, nil, nil, nil)
require.NoError(t, err)
nodeMgr.Start(0, time.Minute)
defer nodeMgr.Stop()
txMgr := transactions.NewManager(conf, logger, &datastore.NoopWriteLockManager{})
coordinator := NewCoordinator(
testhelper.NewLogger(t),
queue,
repoStore,
nil,
NewNodeManagerRouter(nodeMgr, repoStore),
txMgr,
conf,
protoregistry.GitalyProtoPreregistered,
)
t.Run("forwards accessor operations", func(t *testing.T) {
var primaryChosen int
var secondaryChosen int
for i := 0; i < 16; i++ {
frame, err := proto.Marshal(&gitalypb.FindAllBranchesRequest{Repository: &targetRepo})
require.NoError(t, err)
fullMethod := "/gitaly.RefService/FindAllBranches"
peeker := &mockPeeker{frame: frame}
streamParams, err := coordinator.StreamDirector(correlation.ContextWithCorrelation(ctx, "my-correlation-id"), fullMethod, peeker)
require.NoError(t, err)
require.Contains(t, []string{primaryNodeConf.Address, secondaryNodeConf.Address}, streamParams.Primary().Conn.Target(), "must be redirected to primary or secondary")
var nodeConf config.Node
switch streamParams.Primary().Conn.Target() {
case primaryNodeConf.Address:
nodeConf = primaryNodeConf
primaryChosen++
case secondaryNodeConf.Address:
nodeConf = secondaryNodeConf
secondaryChosen++
}
mi, err := coordinator.registry.LookupMethod(fullMethod)
require.NoError(t, err)
require.Equal(t, protoregistry.OpAccessor, mi.Operation, "method must be an accessor")
m, err := protoMessage(mi, streamParams.Primary().Msg)
require.NoError(t, err)
rewrittenTargetRepo, err := mi.TargetRepo(m)
require.NoError(t, err)
require.Equal(t, nodeConf.Storage, rewrittenTargetRepo.GetStorageName(), "stream director must rewrite the storage name")
}
require.NotZero(t, primaryChosen, "primary should have been chosen at least once")
require.NotZero(t, secondaryChosen, "secondary should have been chosen at least once")
})
t.Run("forwards accessor to primary if force-routing", func(t *testing.T) {
var primaryChosen int
var secondaryChosen int
for i := 0; i < 16; i++ {
frame, err := proto.Marshal(&gitalypb.FindAllBranchesRequest{Repository: &targetRepo})
require.NoError(t, err)
fullMethod := "/gitaly.RefService/FindAllBranches"
peeker := &mockPeeker{frame: frame}
ctx := correlation.ContextWithCorrelation(ctx, "my-correlation-id")
ctx = testhelper.MergeIncomingMetadata(ctx, metadata.Pairs(routeRepositoryAccessorPolicy, routeRepositoryAccessorPolicyPrimaryOnly))
streamParams, err := coordinator.StreamDirector(ctx, fullMethod, peeker)
require.NoError(t, err)
require.Contains(t, []string{primaryNodeConf.Address, secondaryNodeConf.Address}, streamParams.Primary().Conn.Target(), "must be redirected to primary or secondary")
var nodeConf config.Node
switch streamParams.Primary().Conn.Target() {
case primaryNodeConf.Address:
nodeConf = primaryNodeConf
primaryChosen++
case secondaryNodeConf.Address:
nodeConf = secondaryNodeConf
secondaryChosen++
}
mi, err := coordinator.registry.LookupMethod(fullMethod)
require.NoError(t, err)
require.Equal(t, protoregistry.OpAccessor, mi.Operation, "method must be an accessor")
m, err := protoMessage(mi, streamParams.Primary().Msg)
require.NoError(t, err)
rewrittenTargetRepo, err := mi.TargetRepo(m)
require.NoError(t, err)
require.Equal(t, nodeConf.Storage, rewrittenTargetRepo.GetStorageName(), "stream director must rewrite the storage name")
}
require.Equal(t, 16, primaryChosen, "primary should have always been chosen")
require.Zero(t, secondaryChosen, "secondary should never have been chosen")
})
t.Run("forwards accessor to primary for primary-only RPCs", func(t *testing.T) {
var primaryChosen int
var secondaryChosen int
for i := 0; i < 16; i++ {
frame, err := proto.Marshal(&gitalypb.GetObjectDirectorySizeRequest{Repository: &targetRepo})
require.NoError(t, err)
fullMethod := "/gitaly.RepositoryService/GetObjectDirectorySize"
peeker := &mockPeeker{frame: frame}
ctx := testhelper.Context(t)
streamParams, err := coordinator.StreamDirector(ctx, fullMethod, peeker)
require.NoError(t, err)
require.Contains(t, []string{primaryNodeConf.Address, secondaryNodeConf.Address}, streamParams.Primary().Conn.Target(), "must be redirected to primary or secondary")
var nodeConf config.Node
switch streamParams.Primary().Conn.Target() {
case primaryNodeConf.Address:
nodeConf = primaryNodeConf
primaryChosen++
case secondaryNodeConf.Address:
nodeConf = secondaryNodeConf
secondaryChosen++
}
mi, err := coordinator.registry.LookupMethod(fullMethod)
require.NoError(t, err)
require.Equal(t, protoregistry.OpAccessor, mi.Operation, "method must be an accessor")
m, err := protoMessage(mi, streamParams.Primary().Msg)
require.NoError(t, err)
rewrittenTargetRepo, err := mi.TargetRepo(m)
require.NoError(t, err)
require.Equal(t, nodeConf.Storage, rewrittenTargetRepo.GetStorageName(), "stream director must rewrite the storage name")
}
require.Equal(t, 16, primaryChosen, "primary should have always been chosen")
require.Zero(t, secondaryChosen, "secondary should never have been chosen")
})
t.Run("forwards accessor operations only to healthy nodes", func(t *testing.T) {
healthSrv.SetServingStatus("", grpc_health_v1.HealthCheckResponse_NOT_SERVING)
shard, err := nodeMgr.GetShard(ctx, conf.VirtualStorages[0].Name)
require.NoError(t, err)
gitaly1, err := shard.GetNode(secondaryNodeConf.Storage)
require.NoError(t, err)
waitNodeToChangeHealthStatus(t, ctx, gitaly1, false)
defer func() {
healthSrv.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)
waitNodeToChangeHealthStatus(t, ctx, gitaly1, true)
}()
frame, err := proto.Marshal(&gitalypb.FindAllBranchesRequest{Repository: &targetRepo})
require.NoError(t, err)
fullMethod := "/gitaly.RefService/FindAllBranches"
peeker := &mockPeeker{frame: frame}
streamParams, err := coordinator.StreamDirector(correlation.ContextWithCorrelation(ctx, "my-correlation-id"), fullMethod, peeker)
require.NoError(t, err)
require.Equal(t, primaryNodeConf.Address, streamParams.Primary().Conn.Target(), "must be redirected to primary")
mi, err := coordinator.registry.LookupMethod(fullMethod)
require.NoError(t, err)
require.Equal(t, protoregistry.OpAccessor, mi.Operation, "method must be an accessor")
m, err := protoMessage(mi, streamParams.Primary().Msg)
require.NoError(t, err)
rewrittenTargetRepo, err := mi.TargetRepo(m)
require.NoError(t, err)
require.Equal(t, "gitaly-1", rewrittenTargetRepo.GetStorageName(), "stream director must rewrite the storage name")
})
t.Run("fails if force-routing to unhealthy primary", func(t *testing.T) {
primaryHealthSrv.SetServingStatus("", grpc_health_v1.HealthCheckResponse_NOT_SERVING)
shard, err := nodeMgr.GetShard(ctx, conf.VirtualStorages[0].Name)
require.NoError(t, err)
primaryGitaly, err := shard.GetNode(primaryNodeConf.Storage)
require.NoError(t, err)
waitNodeToChangeHealthStatus(t, ctx, primaryGitaly, false)
defer func() {
primaryHealthSrv.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)
waitNodeToChangeHealthStatus(t, ctx, primaryGitaly, true)
}()
frame, err := proto.Marshal(&gitalypb.FindAllBranchesRequest{Repository: &targetRepo})
require.NoError(t, err)
fullMethod := "/gitaly.RefService/FindAllBranches"
ctx := correlation.ContextWithCorrelation(ctx, "my-correlation-id")
ctx = testhelper.MergeIncomingMetadata(ctx, metadata.Pairs(routeRepositoryAccessorPolicy, routeRepositoryAccessorPolicyPrimaryOnly))
peeker := &mockPeeker{frame: frame}
_, err = coordinator.StreamDirector(ctx, fullMethod, peeker)
require.True(t, errors.Is(err, nodes.ErrPrimaryNotHealthy))
})
t.Run("doesn't forward mutator operations", func(t *testing.T) {
frame, err := proto.Marshal(&gitalypb.UserUpdateBranchRequest{Repository: &targetRepo})
require.NoError(t, err)
fullMethod := "/gitaly.OperationService/UserUpdateBranch"
peeker := &mockPeeker{frame: frame}
streamParams, err := coordinator.StreamDirector(correlation.ContextWithCorrelation(ctx, "my-correlation-id"), fullMethod, peeker)
require.NoError(t, err)
require.Equal(t, primaryNodeConf.Address, streamParams.Primary().Conn.Target(), "must be redirected to primary")
mi, err := coordinator.registry.LookupMethod(fullMethod)
require.NoError(t, err)
require.Equal(t, protoregistry.OpMutator, mi.Operation, "method must be a mutator")
m, err := protoMessage(mi, streamParams.Primary().Msg)
require.NoError(t, err)
rewrittenTargetRepo, err := mi.TargetRepo(m)
require.NoError(t, err)
require.Equal(t, "gitaly-1", rewrittenTargetRepo.GetStorageName(), "stream director must rewrite the storage name")
})
}
func TestRewrittenRepositoryMessage(t *testing.T) {
t.Parallel()
buildRequest := func(storageName, relativePath, additionalRelativePath string) *gitalypb.CreateObjectPoolRequest {
return &gitalypb.CreateObjectPoolRequest{
ObjectPool: &gitalypb.ObjectPool{
Repository: &gitalypb.Repository{
StorageName: storageName,
RelativePath: relativePath,
},
},
Origin: &gitalypb.Repository{
StorageName: storageName,
RelativePath: additionalRelativePath,
},
}
}
t.Run("successfully rewritten request", func(t *testing.T) {
originalRequest := buildRequest("original-storage", "original-relative-path", "original-additional-relative-path")
methodInfo, err := protoregistry.GitalyProtoPreregistered.LookupMethod("/gitaly.ObjectPoolService/CreateObjectPool")
require.NoError(t, err)
rewrittenMessageBytes, err := rewrittenRepositoryMessage(methodInfo, originalRequest, "rewritten-storage", "rewritten-relative-path", "rewritten-additional-relative-path")
require.NoError(t, err)
var rewrittenMessage gitalypb.CreateObjectPoolRequest
require.NoError(t, proto.Unmarshal(rewrittenMessageBytes, &rewrittenMessage))
testhelper.ProtoEqual(t, buildRequest("original-storage", "original-relative-path", "original-additional-relative-path"), originalRequest)
testhelper.ProtoEqual(t, buildRequest("rewritten-storage", "rewritten-relative-path", "rewritten-additional-relative-path"), &rewrittenMessage)
})
t.Run("rewriting differing storages fails", func(t *testing.T) {
request := &gitalypb.CreateObjectPoolRequest{
ObjectPool: &gitalypb.ObjectPool{
Repository: &gitalypb.Repository{
StorageName: "a",
RelativePath: "a",
},
},
Origin: &gitalypb.Repository{
StorageName: "b",
RelativePath: "b",
},
}
methodInfo, err := protoregistry.GitalyProtoPreregistered.LookupMethod("/gitaly.ObjectPoolService/CreateObjectPool")
require.NoError(t, err)
rewrittenRequest, err := rewrittenRepositoryMessage(methodInfo, request, "rewritten-storage", "rewritten-relative-path", "rewritten-additional-relative-path")
require.Equal(t, structerr.NewInvalidArgument("resolving additional repository on different storage than target repository is not supported"), err)
require.Nil(t, rewrittenRequest)
})
}
func TestStreamDirector_repo_creation(t *testing.T) {
t.Parallel()
db := testdb.New(t)
for i, tc := range []struct {
desc string
replicationFactor int
primaryStored string
assignmentsStored int
mockRepositoryStore func() datastore.RepositoryStore
expectedErr error
}{
{
desc: "without variable replication factor",
primaryStored: "praefect-internal-1",
assignmentsStored: 0,
},
{
desc: "with variable replication factor",
replicationFactor: 3,
primaryStored: "praefect-internal-1",
assignmentsStored: 3,
},
{
desc: "repository metadata already exists",
mockRepositoryStore: func() datastore.RepositoryStore {
return &datastore.MockRepositoryStore{
CreateRepositoryFunc: func(context.Context, int64, string, string, string, string, []string, []string, bool, bool) error {
return datastore.ErrRepositoryAlreadyExists
},
}
},
expectedErr: structerr.NewAlreadyExists("%w", storage.ErrRepositoryAlreadyExists),
},
} {
t.Run(tc.desc, func(t *testing.T) {
tx := db.Begin(t)
defer tx.Rollback(t)
primaryNode := &config.Node{Storage: "praefect-internal-1"}
healthySecondaryNode := &config.Node{Storage: "praefect-internal-2"}
unhealthySecondaryNode := &config.Node{Storage: "praefect-internal-3"}
conf := config.Config{
Failover: config.Failover{ElectionStrategy: config.ElectionStrategyPerRepository},
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
DefaultReplicationFactor: tc.replicationFactor,
Nodes: []*config.Node{primaryNode, healthySecondaryNode, unhealthySecondaryNode},
},
},
}
rewrittenStorage := primaryNode.Storage
targetRepo := gitalypb.Repository{
StorageName: "praefect",
RelativePath: "/path/to/hashed/storage",
}
conns := Connections{
"praefect": {
primaryNode.Storage: &grpc.ClientConn{},
healthySecondaryNode.Storage: &grpc.ClientConn{},
unhealthySecondaryNode.Storage: &grpc.ClientConn{},
},
}
primaryConnPointer := fmt.Sprintf("%p", conns["praefect"][primaryNode.Storage])
secondaryConnPointers := []string{fmt.Sprintf("%p", conns["praefect"][healthySecondaryNode.Storage])}
var repositoryStore datastore.RepositoryStore = datastore.NewPostgresRepositoryStore(tx, conf.StorageNames())
if tc.mockRepositoryStore != nil {
repositoryStore = tc.mockRepositoryStore()
}
router := NewPerRepositoryRouter(
conns,
nil,
StaticHealthChecker{"praefect": {primaryNode.Storage, healthySecondaryNode.Storage}},
mockRandom{
intnFunc: func(n int) int {
require.Equal(t, n, 2, "number of primary candidates should match the number of healthy nodes")
return 0
},
shuffleFunc: func(n int, swap func(int, int)) {
require.Equal(t, n, 2, "number of secondary candidates should match the number of node minus the primary")
},
},
nil,
nil,
repositoryStore,
conf.DefaultReplicationFactors(),
)
logger := testhelper.NewLogger(t)
ctx := testhelper.Context(t)
repoWriteLockMgr := datastore.NewRepoReferenceWriteLockManager(ctx, db, testdb.GetConfig(t, db.Name), logger)
txMgr := transactions.NewManager(conf, logger, repoWriteLockMgr)
queueInterceptor := datastore.NewReplicationEventQueueInterceptor(datastore.NewPostgresReplicationEventQueue(tx))
coordinator := NewCoordinator(
logger,
queueInterceptor,
repositoryStore,
nil,
router,
txMgr,
conf,
protoregistry.GitalyProtoPreregistered,
)
frame, err := proto.Marshal(&gitalypb.CreateRepositoryRequest{
Repository: &targetRepo,
})
require.NoError(t, err)
fullMethod := "/gitaly.RepositoryService/CreateRepository"
peeker := &mockPeeker{frame}
streamParams, err := coordinator.StreamDirector(correlation.ContextWithCorrelation(ctx, "my-correlation-id"), fullMethod, peeker)
require.NoError(t, err)
require.Equal(t, primaryConnPointer, fmt.Sprintf("%p", streamParams.Primary().Conn))
var secondaries []string
for _, dst := range streamParams.Secondaries() {
secondaries = append(secondaries, fmt.Sprintf("%p", dst.Conn))
}
require.Equal(t, secondaryConnPointers, secondaries, "secondary connections did not match expected")
mi, err := coordinator.registry.LookupMethod(fullMethod)
require.NoError(t, err)
m, err := mi.UnmarshalRequestProto(streamParams.Primary().Msg)
require.NoError(t, err)
rewrittenTargetRepo, err := mi.TargetRepo(m)
require.NoError(t, err)
require.Equal(t, rewrittenStorage, rewrittenTargetRepo.GetStorageName(), "stream director should have rewritten the storage name")
vote := voting.VoteFromData([]byte{})
require.NoError(t, txMgr.VoteTransaction(ctx, 1, rewrittenTargetRepo.GetStorageName(), rewrittenTargetRepo.GetRelativePath(),
"praefect-internal-1", gitalypb.VoteTransactionRequest_PREPARING_PHASE, vote))
require.NoError(t, txMgr.VoteTransaction(ctx, 1, rewrittenTargetRepo.GetStorageName(), rewrittenTargetRepo.GetRelativePath(),
"praefect-internal-2", gitalypb.VoteTransactionRequest_PREPARING_PHASE, vote))
// this call creates new events in the queue and simulates usual flow of the update operation
err = streamParams.RequestFinalizer()
require.Equal(t, tc.expectedErr, err)
if tc.expectedErr != nil {
return
}
// wait until event persisted (async operation)
require.NoError(t, queueInterceptor.Wait(time.Minute, func(i *datastore.ReplicationEventQueueInterceptor) bool {
return len(i.GetEnqueuedResult()) == 1
}))
var expectedEvents, actualEvents []datastore.ReplicationEvent
for _, target := range []string{unhealthySecondaryNode.Storage} {
actual, err := queueInterceptor.Dequeue(ctx, "praefect", target, 10)
require.NoError(t, err)
require.Len(t, actual, 1)
actualEvents = append(actualEvents, actual[0])
expectedEvents = append(expectedEvents, datastore.ReplicationEvent{
ID: actual[0].ID,
State: datastore.JobStateInProgress,
Attempt: 2,
LockID: fmt.Sprintf("praefect|%s|/path/to/hashed/storage", target),
CreatedAt: actual[0].CreatedAt,
UpdatedAt: actual[0].UpdatedAt,
Job: datastore.ReplicationJob{
RepositoryID: int64(i + 1),
Change: datastore.UpdateRepo,
VirtualStorage: conf.VirtualStorages[0].Name,
RelativePath: targetRepo.GetRelativePath(),
TargetNodeStorage: target,
SourceNodeStorage: primaryNode.Storage,
},
Meta: datastore.Params{datastore.CorrelationIDKey: "my-correlation-id"},
})
}
require.Equal(t, expectedEvents, actualEvents, "ensure replication job created by stream director is correct")
// Verify that primary was stored or not according to tc.primaryStored
var primaryStorage string
require.NoError(t, tx.QueryRowContext(ctx,
`SELECT "primary" FROM repositories WHERE relative_path = $1`,
targetRepo.GetRelativePath(),
).Scan(&primaryStorage))
require.Equal(t, tc.primaryStored, primaryStorage)
// Verify that repository assignments were stored or not according to tc.assignmentsStored
var assignmentCount int
require.NoError(t, tx.QueryRowContext(ctx,
"SELECT COUNT(*) FROM repository_assignments WHERE relative_path = $1",
targetRepo.GetRelativePath(),
).Scan(&assignmentCount))
require.Equal(t, tc.assignmentsStored, assignmentCount)
})
}
}
// TestStreamDirector_additional_repo_creation tests datastore.CreateRepo requests which include an
// additionalReplicaPath argument. This is true for RPCs like CreateFork and CreateObjectPool, where we must consult
// the source repository when creating the new target repository.
//
// These tests ensure that if the source repository had entries in the repository_assignments table, that these entries
// are set for the target repository too. This is important for forks and object pools, since an entire object pool
// network must reside on the disks of the same subset of replicas.
func TestStreamDirector_additional_repo_creation(t *testing.T) {
t.Parallel()
db := testdb.New(t)
for _, tc := range []struct {
desc string
fullMethod string
createRequest func(targetRelativePath, sourceRelativePath string) proto.Message
targetRelativePath string
sourceHasAssignments bool
expectedNewRepoAssignments []string
}{
{
desc: "CreateFork without source assignments",
fullMethod: "/gitaly.RepositoryService/CreateFork",
createRequest: func(targetRelativePath, sourceRelativePath string) proto.Message {
return &gitalypb.CreateForkRequest{
Repository: &gitalypb.Repository{
StorageName: "praefect",
RelativePath: targetRelativePath,
},
SourceRepository: &gitalypb.Repository{
StorageName: "praefect",
RelativePath: sourceRelativePath,
},
}
},
targetRelativePath: "/path/to/forked/repo",
sourceHasAssignments: false,
},
{
desc: "CreateFork with source assignments",
fullMethod: "/gitaly.RepositoryService/CreateFork",
createRequest: func(targetRelativePath, sourceRelativePath string) proto.Message {
return &gitalypb.CreateForkRequest{
Repository: &gitalypb.Repository{
StorageName: "praefect",
RelativePath: targetRelativePath,
},
SourceRepository: &gitalypb.Repository{
StorageName: "praefect",
RelativePath: sourceRelativePath,
},
}
},
targetRelativePath: "/path/to/forked/repo",
sourceHasAssignments: true,
expectedNewRepoAssignments: []string{"praefect-internal-1", "praefect-internal-2"},
},
{
desc: "CreateObjectPool without source assignments",
fullMethod: "/gitaly.ObjectPoolService/CreateObjectPool",
createRequest: func(targetRelativePath, sourceRelativePath string) proto.Message {
return &gitalypb.CreateObjectPoolRequest{
ObjectPool: &gitalypb.ObjectPool{
Repository: &gitalypb.Repository{
StorageName: "praefect",
RelativePath: targetRelativePath,
},
},
Origin: &gitalypb.Repository{
StorageName: "praefect",
RelativePath: sourceRelativePath,
},
}
},
targetRelativePath: "@pools/ab/cd/1234",
sourceHasAssignments: false,
},
{
desc: "CreateObjectPool with source assignments",
fullMethod: "/gitaly.ObjectPoolService/CreateObjectPool",
createRequest: func(targetRelativePath, sourceRelativePath string) proto.Message {
return &gitalypb.CreateObjectPoolRequest{
ObjectPool: &gitalypb.ObjectPool{
Repository: &gitalypb.Repository{
StorageName: "praefect",
RelativePath: targetRelativePath,
},
},
Origin: &gitalypb.Repository{
StorageName: "praefect",
RelativePath: sourceRelativePath,
},
}
},
targetRelativePath: "@pools/ab/cd/1234",
sourceHasAssignments: true,
expectedNewRepoAssignments: []string{"praefect-internal-1", "praefect-internal-2"},
},
} {
t.Run(tc.desc, func(t *testing.T) {
ctx := testhelper.Context(t)
logger := testhelper.NewLogger(t)
// We can't start a DB transaction here and roll back the changes at the end of each test,
// because the CreateObjectPool RPC is designated as non-transactional. Due to this, the
// coordinator marks all secondary replicas as outdated, causing newRequestFinalizer() to
// enqueue replication jobs for all secondaries. Each enqueue happens in its own goroutine,
// and this is problematic multiple goroutines attempt to use the same *sql.Tx instance.
//
// Instead, truncate all data in the test DB before each test. It's probably equivalent, but
// just slower.
db.TruncateAll(t)
primaryNode := &config.Node{Storage: "praefect-internal-1"}
secondaryNode1 := &config.Node{Storage: "praefect-internal-2"}
secondaryNode2 := &config.Node{Storage: "praefect-internal-3"}
conf := config.Config{
Failover: config.Failover{ElectionStrategy: config.ElectionStrategyPerRepository},
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{primaryNode, secondaryNode1, secondaryNode2},
},
},
}
repositoryStore := datastore.NewPostgresRepositoryStore(db, conf.StorageNames())
sourceRelativePath := "@hashed/foo/bar"
sourceReplicaPath := "@cluster/foo/bar"
sourceRepoID, err := repositoryStore.ReserveRepositoryID(ctx, "praefect", sourceRelativePath)
require.NoError(t, err)
// Create source repository with only primary and secondaryNode1.
// When sourceHasAssignments is true, this creates explicit assignments
// to just these two nodes (not secondaryNode2).
require.NoError(t, repositoryStore.CreateRepository(ctx,
sourceRepoID,
"praefect",
sourceRelativePath,
sourceReplicaPath,
primaryNode.Storage,
[]string{secondaryNode1.Storage},
nil,
true,
tc.sourceHasAssignments,
))
conns := Connections{
"praefect": {
primaryNode.Storage: &grpc.ClientConn{},
secondaryNode1.Storage: &grpc.ClientConn{},
secondaryNode2.Storage: &grpc.ClientConn{},
},
}
router := NewPerRepositoryRouter(
conns,
nil,
StaticHealthChecker{"praefect": {primaryNode.Storage, secondaryNode1.Storage, secondaryNode2.Storage}},
nil,
nil,
nil,
repositoryStore,
conf.DefaultReplicationFactors(),
)
repoWriteLockMgr := datastore.NewRepoReferenceWriteLockManager(ctx, db, testdb.GetConfig(t, db.Name), logger)
coordinator := NewCoordinator(
logger,
datastore.NewReplicationEventQueueInterceptor(datastore.NewPostgresReplicationEventQueue(db)),
repositoryStore,
nil,
router,
transactions.NewManager(conf, logger, repoWriteLockMgr),
conf,
protoregistry.GitalyProtoPreregistered,
)
frame, err := proto.Marshal(tc.createRequest(tc.targetRelativePath, sourceRelativePath))
require.NoError(t, err)
streamParams, err := coordinator.StreamDirector(correlation.ContextWithCorrelation(ctx, "my-correlation-id"), tc.fullMethod, &mockPeeker{frame})
require.NoError(t, err)
require.NoError(t, streamParams.RequestFinalizer())
rows, err := db.QueryContext(ctx,
"SELECT storage FROM repository_assignments WHERE relative_path = $1 ORDER BY storage",
tc.targetRelativePath,
)
require.NoError(t, err)
var actualAssignments []string
for rows.Next() {
var st string
require.NoError(t, rows.Scan(&st))
actualAssignments = append(actualAssignments, st)
}
require.NoError(t, rows.Err())
require.NoError(t, rows.Close())
require.ElementsMatch(t, tc.expectedNewRepoAssignments, actualAssignments)
})
}
}
func waitNodeToChangeHealthStatus(t *testing.T, ctx context.Context, node nodes.Node, health bool) {
t.Helper()
require.Eventually(t, func() bool {
if node.IsHealthy() == health {
return true
}
_, err := node.CheckHealth(ctx)
require.NoError(t, err)
return false
}, time.Minute, time.Nanosecond)
}
type mockPeeker struct {
frame []byte
}
func (m *mockPeeker) Peek() ([]byte, error) {
return m.frame, nil
}
func (m *mockPeeker) Modify(payload []byte) error {
m.frame = payload
return nil
}
func TestAbsentCorrelationID(t *testing.T) {
t.Parallel()
gitalySocket0, gitalySocket1 := testhelper.GetTemporaryGitalySocketFileName(t), testhelper.GetTemporaryGitalySocketFileName(t)
healthSrv0 := testhelper.NewServerWithHealth(t, gitalySocket0)
healthSrv1 := testhelper.NewServerWithHealth(t, gitalySocket1)
healthSrv0.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)
healthSrv1.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)
primaryAddress, secondaryAddress := "unix://"+gitalySocket0, "unix://"+gitalySocket1
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{
{
Address: primaryAddress,
Storage: "praefect-internal-1",
},
{
Address: secondaryAddress,
Storage: "praefect-internal-2",
},
},
},
},
}
queueInterceptor := datastore.NewReplicationEventQueueInterceptor(datastore.NewPostgresReplicationEventQueue(testdb.New(t)))
queueInterceptor.OnEnqueue(func(ctx context.Context, event datastore.ReplicationEvent, queue datastore.ReplicationEventQueue) (datastore.ReplicationEvent, error) {
assert.True(t, len(queueInterceptor.GetEnqueued()) < 2, "expected only one event to be created")
return queue.Enqueue(ctx, event)
})
targetRepo := gitalypb.Repository{
StorageName: "praefect",
RelativePath: "/path/to/hashed/storage",
}
ctx := testhelper.Context(t)
logger := testhelper.SharedLogger(t)
nodeMgr, err := nodes.NewManager(logger, conf, nil, nil, promtest.NewMockHistogramVec(), protoregistry.GitalyProtoPreregistered, nil, nil, nil)
require.NoError(t, err)
nodeMgr.Start(0, time.Hour)
defer nodeMgr.Stop()
txMgr := transactions.NewManager(conf, logger, &datastore.NoopWriteLockManager{})
rs := datastore.MockRepositoryStore{}
coordinator := NewCoordinator(
testhelper.NewLogger(t),
queueInterceptor,
rs,
nil,
NewNodeManagerRouter(nodeMgr, rs),
txMgr,
conf,
protoregistry.GitalyProtoPreregistered,
)
frame, err := proto.Marshal(&gitalypb.CreateObjectPoolRequest{
Origin: &targetRepo,
ObjectPool: &gitalypb.ObjectPool{Repository: &targetRepo},
})
require.NoError(t, err)
fullMethod := "/gitaly.ObjectPoolService/CreateObjectPool"
peeker := &mockPeeker{frame}
streamParams, err := coordinator.StreamDirector(ctx, fullMethod, peeker)
require.NoError(t, err)
require.Equal(t, primaryAddress, streamParams.Primary().Conn.Target())
// must be run as it adds replication events to the queue
require.NoError(t, streamParams.RequestFinalizer())
require.NoError(t, queueInterceptor.Wait(time.Minute, func(i *datastore.ReplicationEventQueueInterceptor) bool {
return len(i.GetEnqueuedResult()) == 1
}))
jobs, err := queueInterceptor.Dequeue(ctx, conf.VirtualStorages[0].Name, conf.VirtualStorages[0].Nodes[1].Storage, 1)
require.NoError(t, err)
require.Len(t, jobs, 1)
require.NotZero(t, jobs[0].Meta[datastore.CorrelationIDKey],
"the coordinator should have generated a random ID")
}
func TestCoordinatorEnqueueFailure(t *testing.T) {
testhelper.SkipWithMacOS(t, "this test is extremely flaky on macos platform")
t.Parallel()
conf := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: "praefect",
Nodes: []*config.Node{
{
Address: "unix:///woof",
Storage: "praefect-internal-1",
},
{
Address: "unix:///meow",
Storage: "praefect-internal-2",
},
},
},
},
}
queueInterceptor := datastore.NewReplicationEventQueueInterceptor(nil)
errQ := make(chan error, 1)
queueInterceptor.OnEnqueue(func(ctx context.Context, event datastore.ReplicationEvent, queue datastore.ReplicationEventQueue) (datastore.ReplicationEvent, error) {
return datastore.ReplicationEvent{}, <-errQ
})
queueInterceptor.OnDequeue(func(context.Context, string, string, int, datastore.ReplicationEventQueue) ([]datastore.ReplicationEvent, error) {
return nil, nil
})
queueInterceptor.OnAcknowledge(func(context.Context, datastore.JobState, []uint64, datastore.ReplicationEventQueue) ([]uint64, error) {
return nil, nil
})
registrar := func(srv *grpc.Server) {
gitalypb.RegisterRepositoryServiceServer(srv, &mockRepositoryService{
ReplicateRepositoryFunc: func(context.Context, *gitalypb.ReplicateRepositoryRequest) (*gitalypb.ReplicateRepositoryResponse, error) {
return &gitalypb.ReplicateRepositoryResponse{}, nil
},
})
}
ctx := testhelper.Context(t)
cc, _, cleanup := RunPraefectServer(t, ctx, conf, BuildOptions{
WithAnnotations: protoregistry.GitalyProtoPreregistered,
WithQueue: queueInterceptor,
WithBackends: WithMockBackends(t, map[string]func(*grpc.Server){
conf.VirtualStorages[0].Nodes[0].Storage: registrar,
conf.VirtualStorages[0].Nodes[1].Storage: registrar,
}),
})
defer cleanup()
mcli := gitalypb.NewRepositoryServiceClient(cc)
errQ <- nil
repoReq := &gitalypb.ReplicateRepositoryRequest{
Repository: &gitalypb.Repository{
RelativePath: "meow",
StorageName: conf.VirtualStorages[0].Name,
},
}
_, err := mcli.ReplicateRepository(ctx, repoReq)
require.NoError(t, err)
expectErrMsg := "enqueue failed"
errQ <- errors.New(expectErrMsg)
_, err = mcli.ReplicateRepository(ctx, repoReq)
require.Error(t, err)
require.Equal(t, err.Error(), "rpc error: code = Internal desc = enqueue replication event: "+expectErrMsg)
}
func TestStreamDirectorStorageScope(t *testing.T) {
// stubs health-check requests because nodes.NewManager establishes connection on creation
gitalySocket0, gitalySocket1 := testhelper.GetTemporaryGitalySocketFileName(t), testhelper.GetTemporaryGitalySocketFileName(t)
testhelper.NewServerWithHealth(t, gitalySocket0)
testhelper.NewServerWithHealth(t, gitalySocket1)
primaryAddress, secondaryAddress := "unix://"+gitalySocket0, "unix://"+gitalySocket1
primaryGitaly := &config.Node{Address: primaryAddress, Storage: "gitaly-1"}
secondaryGitaly := &config.Node{Address: secondaryAddress, Storage: "gitaly-2"}
conf := config.Config{
Failover: config.Failover{Enabled: true},
VirtualStorages: []*config.VirtualStorage{{
Name: "praefect",
Nodes: []*config.Node{primaryGitaly, secondaryGitaly},
}},
}
rs := datastore.MockRepositoryStore{}
nodeMgr, err := nodes.NewManager(testhelper.SharedLogger(t), conf, nil, nil, promtest.NewMockHistogramVec(), protoregistry.GitalyProtoPreregistered, nil, nil, nil)
require.NoError(t, err)
nodeMgr.Start(0, time.Second)
defer nodeMgr.Stop()
coordinator := NewCoordinator(
testhelper.NewLogger(t),
nil,
rs,
nil,
NewNodeManagerRouter(nodeMgr, rs),
nil,
conf,
protoregistry.GitalyProtoPreregistered,
)
ctx := testhelper.Context(t)
t.Run("accessor", func(t *testing.T) {
fullMethod := "/gitaly.InternalGitaly/WalkRepos"
requireScopeOperation(t, coordinator.registry, fullMethod, protoregistry.ScopeStorage, protoregistry.OpAccessor)
frame, err := proto.Marshal(&gitalypb.WalkReposRequest{
StorageName: conf.VirtualStorages[0].Name,
})
require.NoError(t, err)
streamParams, err := coordinator.StreamDirector(ctx, fullMethod, &mockPeeker{frame})
require.NoError(t, err)
require.Equal(t, primaryAddress, streamParams.Primary().Conn.Target(), "stream director didn't redirect to gitaly storage")
rewritten := gitalypb.WalkReposRequest{}
require.NoError(t, proto.Unmarshal(streamParams.Primary().Msg, &rewritten))
require.Equal(t, primaryGitaly.Storage, rewritten.GetStorageName(), "stream director didn't rewrite storage")
})
}
func TestStreamDirectorStorageScopeError(t *testing.T) {
ctx := testhelper.Context(t)
t.Run("no storage provided", func(t *testing.T) {
mgr := &nodes.MockManager{
GetShardFunc: func(s string) (nodes.Shard, error) {
require.FailNow(t, "validation of input was not executed")
return nodes.Shard{}, assert.AnError
},
}
rs := datastore.MockRepositoryStore{}
coordinator := NewCoordinator(
testhelper.NewLogger(t),
nil,
rs,
nil,
NewNodeManagerRouter(mgr, rs),
nil,
config.Config{},
protoregistry.GitalyProtoPreregistered,
)
frame, err := proto.Marshal(&gitalypb.WalkReposRequest{StorageName: ""})
require.NoError(t, err)
_, err = coordinator.StreamDirector(ctx, "/gitaly.InternalGitaly/WalkRepos", &mockPeeker{frame})
require.Error(t, err)
result, ok := status.FromError(err)
require.True(t, ok)
require.Equal(t, codes.InvalidArgument, result.Code())
require.Equal(t, "storage scoped: target storage field not found", result.Message())
})
t.Run("unknown storage provided", func(t *testing.T) {
mgr := &nodes.MockManager{
GetShardFunc: func(s string) (nodes.Shard, error) {
require.Equal(t, "fake", s)
return nodes.Shard{}, nodes.ErrVirtualStorageNotExist
},
}
rs := datastore.MockRepositoryStore{}
coordinator := NewCoordinator(
testhelper.NewLogger(t),
nil,
rs,
nil,
NewNodeManagerRouter(mgr, rs),
nil,
config.Config{},
protoregistry.GitalyProtoPreregistered,
)
frame, err := proto.Marshal(&gitalypb.WalkReposRequest{StorageName: "fake"})
require.NoError(t, err)
_, err = coordinator.StreamDirector(ctx, "/gitaly.InternalGitaly/WalkRepos", &mockPeeker{frame})
require.Error(t, err)
result, ok := status.FromError(err)
require.True(t, ok)
require.Equal(t, codes.InvalidArgument, result.Code())
require.Equal(t, "virtual storage does not exist", result.Message())
})
t.Run("primary gitaly is not healthy", func(t *testing.T) {
t.Run("accessor", func(t *testing.T) {
mgr := &nodes.MockManager{
GetShardFunc: func(s string) (nodes.Shard, error) {
require.Equal(t, "fake", s)
return nodes.Shard{}, nodes.ErrPrimaryNotHealthy
},
}
rs := datastore.MockRepositoryStore{}
coordinator := NewCoordinator(
testhelper.NewLogger(t),
nil,
rs,
nil,
NewNodeManagerRouter(mgr, rs),
nil,
config.Config{},
protoregistry.GitalyProtoPreregistered,
)
fullMethod := "/gitaly.InternalGitaly/WalkRepos"
requireScopeOperation(t, coordinator.registry, fullMethod, protoregistry.ScopeStorage, protoregistry.OpAccessor)
frame, err := proto.Marshal(&gitalypb.WalkReposRequest{StorageName: "fake"})
require.NoError(t, err)
_, err = coordinator.StreamDirector(ctx, fullMethod, &mockPeeker{frame})
require.Error(t, err)
result, ok := status.FromError(err)
require.True(t, ok)
require.Equal(t, codes.Internal, result.Code())
require.Equal(t, `accessor storage scoped: route storage accessor "fake": primary gitaly is not healthy`, result.Message())
})
})
}
func TestDisabledTransactionsWithFeatureFlag(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for rpc, enabledFn := range transactionRPCs {
if enabledFn(ctx) {
require.True(t, shouldUseTransaction(ctx, rpc))
}
}
}
func requireScopeOperation(t *testing.T, registry *protoregistry.Registry, fullMethod string, scope protoregistry.Scope, op protoregistry.OpType) {
t.Helper()
mi, err := registry.LookupMethod(fullMethod)
require.NoError(t, err)
require.Equal(t, scope, mi.Scope, "scope doesn't match requested")
require.Equal(t, op, mi.Operation, "operation type doesn't match requested")
}
type mockOperationServer struct {
gitalypb.UnimplementedOperationServiceServer
t testing.TB
wg *sync.WaitGroup
err error
called bool
}
func (s *mockOperationServer) UserCreateBranch(
context.Context,
*gitalypb.UserCreateBranchRequest,
) (*gitalypb.UserCreateBranchResponse, error) {
// We need to wait for all servers to arrive in this RPC. If we don't it could be that for
// example the primary arrives quicker than the others and directly errors. This would cause
// stream cancellation, and if the secondaries didn't yet end up in UserCreateBranch, we
// wouldn't see the function call.
s.called = true
s.wg.Done()
s.wg.Wait()
return &gitalypb.UserCreateBranchResponse{}, s.err
}
type mockLeaseEnder struct{}
func (e mockLeaseEnder) EndLease(context.Context) error {
return nil
}
type mockDiskCache struct {
cache.Cache
}
func (c *mockDiskCache) StartLease(context.Context, *gitalypb.Repository) (cache.LeaseEnder, error) {
return mockLeaseEnder{}, nil
}
// TestCoordinator_grpcErrorHandling asserts that we correctly proxy errors in case any of the nodes
// fails. Most importantly, we want to make sure to only ever forward errors from the primary and
// never from the secondaries.
func TestCoordinator_grpcErrorHandling(t *testing.T) {
t.Parallel()
praefectConfig := config.Config{
VirtualStorages: []*config.VirtualStorage{
{
Name: testhelper.DefaultStorageName,
},
},
}
type gitalyNode struct {
mock *nodes.MockNode
operationServer *mockOperationServer
}
ctx := testhelper.Context(t)
relativePath := gittest.NewRepositoryName(t)
for _, tc := range []struct {
desc string
errByNode map[string]error
expectedErr error
}{
{
desc: "no errors",
},
{
desc: "primary error gets forwarded",
errByNode: map[string]error{
"primary": errors.New("foo"),
},
expectedErr: status.Error(codes.Internal, "foo"),
},
{
desc: "secondary error gets ignored",
errByNode: map[string]error{
"secondary-1": errors.New("foo"),
},
},
{
desc: "primary error has precedence",
errByNode: map[string]error{
"primary": errors.New("primary"),
"secondary-1": errors.New("secondary-1"),
"secondary-2": errors.New("secondary-2"),
},
expectedErr: status.Error(codes.Internal, "primary"),
},
} {
t.Run(tc.desc, func(t *testing.T) {
var wg sync.WaitGroup
gitalies := make(map[string]gitalyNode)
for _, gitaly := range []string{"primary", "secondary-1", "secondary-2"} {
cfg := testcfg.Build(t, testcfg.WithStorages(gitaly))
_, _ = gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
SkipCreationViaService: true,
RelativePath: relativePath,
})
operationServer := &mockOperationServer{
t: t,
wg: &wg,
}
addr := testserver.RunGitalyServer(t, cfg, func(srv *grpc.Server, deps *service.Dependencies) {
gitalypb.RegisterOperationServiceServer(srv, operationServer)
}, testserver.WithDiskCache(&mockDiskCache{}), testserver.WithDisablePraefect())
conn, err := client.New(ctx, addr, client.WithGrpcOptions([]grpc.DialOption{
grpc.WithDefaultCallOptions(grpc.ForceCodec(proxy.NewCodec())),
}))
require.NoError(t, err)
defer conn.Close()
gitalies[gitaly] = gitalyNode{
mock: &nodes.MockNode{
Conn: conn,
Healthy: true,
GetStorageMethod: func() string { return gitaly },
},
operationServer: operationServer,
}
praefectConfig.VirtualStorages[0].Nodes = append(praefectConfig.VirtualStorages[0].Nodes, &config.Node{
Address: addr,
Storage: gitaly,
})
}
praefectConn, _, cleanup := RunPraefectServer(t, ctx, praefectConfig, BuildOptions{
// Set up a mock manager which sets up primary/secondaries and pretends that all nodes are
// healthy. We need fixed roles and unhealthy nodes will not take part in transactions.
WithNodeMgr: &nodes.MockManager{
Storage: testhelper.DefaultStorageName,
GetShardFunc: func(shardName string) (nodes.Shard, error) {
require.Equal(t, testhelper.DefaultStorageName, shardName)
return nodes.Shard{
Primary: gitalies["primary"].mock,
Secondaries: []nodes.Node{
gitalies["secondary-1"].mock,
gitalies["secondary-2"].mock,
},
}, nil
},
},
// Set up a mock repsoitory store pretending that all nodes are consistent. Only consistent
// nodes will take part in transactions.
WithRepoStore: datastore.MockRepositoryStore{
GetReplicaPathFunc: func(ctx context.Context, repositoryID int64) (string, error) {
return relativePath, nil
},
GetConsistentStoragesFunc: func(ctx context.Context, virtualStorage, relativePath string) (string, *datastructure.Set[string], error) {
return relativePath, datastructure.SetFromValues("primary", "secondary-1", "secondary-2"), nil
},
},
})
defer cleanup()
for name, node := range gitalies {
wg.Add(1)
node.operationServer.err = tc.errByNode[name]
node.operationServer.called = false
}
_, err := gitalypb.NewOperationServiceClient(praefectConn).UserCreateBranch(ctx,
&gitalypb.UserCreateBranchRequest{
Repository: &gitalypb.Repository{
StorageName: testhelper.DefaultStorageName,
RelativePath: relativePath,
},
})
testhelper.RequireGrpcError(t, tc.expectedErr, err)
for _, node := range gitalies {
require.True(t, node.operationServer.called, "expected gitaly %q to have been called", node.mock.GetStorage())
}
})
}
}
type mockTransaction struct {
nodeStates map[string]transactions.VoteResult
subtransactions int
didVote map[string]bool
committedConcluded chan struct{}
}
func (t mockTransaction) ID() uint64 {
return 0
}
func (t mockTransaction) CountSubtransactions() int {
return t.subtransactions
}
func (t mockTransaction) DidVote(node string) bool {
return t.didVote[node]
}
func (t mockTransaction) State() (map[string]transactions.VoteResult, error) {
return t.nodeStates, nil
}
func (t mockTransaction) CommittedConcluded() <-chan struct{} {
if t.committedConcluded != nil {
return t.committedConcluded
}
ch := make(chan struct{})
close(ch)
return ch
}
func TestGetUpdatedAndOutdatedSecondaries(t *testing.T) {
type node struct {
name string
state transactions.VoteResult
err error
}
ctx := testhelper.Context(t)
anyErr := errors.New("arbitrary error")
grpcErr := status.Error(codes.Internal, "arbitrary gRPC error")
for _, tc := range []struct {
desc string
primary node
secondaries []node
replicas []string
subtransactions int
didVote map[string]bool
expectedPrimaryDirtied bool
expectedOutdated []string
expectedUpdated []string
expectedMetrics map[string]int
}{
{
desc: "single committed node",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
},
{
desc: "single failed node",
primary: node{
name: "primary",
state: transactions.VoteFailed,
},
subtransactions: 1,
},
{
desc: "single node with standard error",
primary: node{
name: "primary",
err: anyErr,
},
},
{
desc: "single node with gRPC error",
primary: node{
name: "primary",
err: grpcErr,
},
},
{
desc: "single node without subtransactions",
primary: node{
name: "primary",
},
subtransactions: 0,
expectedPrimaryDirtied: true,
},
{
desc: "single successful node with replica",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
},
replicas: []string{"replica"},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"replica"},
expectedMetrics: map[string]int{
"outdated": 1,
},
},
{
desc: "single failing node with replica is not considered modified",
primary: node{
name: "primary",
state: transactions.VoteFailed,
},
subtransactions: 1,
},
{
desc: "single node with standard error with replica",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: anyErr,
},
replicas: []string{"replica"},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"replica"},
expectedMetrics: map[string]int{
"outdated": 1,
},
},
{
desc: "single node with gRPC error with replica",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: grpcErr,
},
replicas: []string{"replica"},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"replica"},
expectedMetrics: map[string]int{
"outdated": 1,
},
},
{
desc: "single node with standard error without commit with replica",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: anyErr,
},
replicas: []string{"replica"},
subtransactions: 1,
expectedPrimaryDirtied: false,
},
{
desc: "single node with gRPC error without commit with replica",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: grpcErr,
},
replicas: []string{"replica"},
subtransactions: 1,
expectedPrimaryDirtied: false,
},
{
desc: "single node without transaction with replica",
primary: node{
name: "primary",
},
replicas: []string{"replica"},
subtransactions: 0,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"replica"},
expectedMetrics: map[string]int{
"outdated": 1,
},
},
{
desc: "multiple committed nodes",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
},
secondaries: []node{
{name: "s1", state: transactions.VoteCommitted},
{name: "s2", state: transactions.VoteCommitted},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedUpdated: []string{"s1", "s2"},
expectedMetrics: map[string]int{
"updated": 2,
},
},
{
desc: "multiple committed nodes with primary err",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: anyErr,
},
secondaries: []node{
{name: "s1", state: transactions.VoteCommitted},
{name: "s2", state: transactions.VoteCommitted},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1", "s2"},
expectedMetrics: map[string]int{
"node-error-status": 2,
},
},
{
desc: "multiple committed nodes with primary gRPC err",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: grpcErr,
},
secondaries: []node{
{name: "s1", state: transactions.VoteCommitted},
{name: "s2", state: transactions.VoteCommitted},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1", "s2"},
expectedMetrics: map[string]int{
"node-error-status": 2,
},
},
{
desc: "multiple committed nodes with same standard error as primary",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: anyErr,
},
secondaries: []node{
{name: "s1", state: transactions.VoteCommitted, err: anyErr},
{name: "s2", state: transactions.VoteCommitted, err: anyErr},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1", "s2"},
expectedMetrics: map[string]int{
"node-error-status": 2,
},
},
{
desc: "multiple committed nodes with same gRPC error as primary",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: grpcErr,
},
secondaries: []node{
{name: "s1", state: transactions.VoteCommitted, err: grpcErr},
{name: "s2", state: transactions.VoteCommitted, err: grpcErr},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedUpdated: []string{"s1", "s2"},
expectedMetrics: map[string]int{
"updated": 2,
},
},
{
desc: "committed node with same generated error code and message",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: status.Error(codes.Internal, "some error"),
},
secondaries: []node{
{
name: "s1",
state: transactions.VoteCommitted,
err: status.Error(codes.Internal, "some error"),
},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedUpdated: []string{"s1"},
expectedMetrics: map[string]int{
"updated": 1,
},
},
{
desc: "committed node with different generated error code",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: status.Error(codes.Internal, "some error"),
},
secondaries: []node{
{
name: "s1",
state: transactions.VoteCommitted,
err: status.Error(codes.FailedPrecondition, "some error"),
},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1"},
expectedMetrics: map[string]int{
"node-error-status": 1,
},
},
{
desc: "committed node with different generated error message",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: status.Error(codes.Internal, "some error"),
},
secondaries: []node{
{
name: "s1",
state: transactions.VoteCommitted,
err: status.Error(codes.Internal, "different error"),
},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1"},
expectedMetrics: map[string]int{
"node-error-status": 1,
},
},
{
desc: "multiple committed nodes with different error as primary",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: grpcErr,
},
secondaries: []node{
{name: "s1", state: transactions.VoteCommitted, err: status.Error(codes.Internal, "somethingsomething")},
{name: "s2", state: transactions.VoteCommitted, err: grpcErr},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedUpdated: []string{"s2"},
expectedOutdated: []string{"s1"},
expectedMetrics: map[string]int{
"node-error-status": 1,
"updated": 1,
},
},
{
desc: "multiple committed nodes with secondary err",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
},
secondaries: []node{
{name: "s1", state: transactions.VoteCommitted, err: grpcErr},
{name: "s2", state: transactions.VoteCommitted},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedUpdated: []string{"s2"},
expectedOutdated: []string{"s1"},
expectedMetrics: map[string]int{
"node-error-status": 1,
"updated": 1,
},
},
{
desc: "multiple committed nodes with primary and missing secondary err",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
err: grpcErr,
},
secondaries: []node{
{name: "s1", state: transactions.VoteCommitted, err: grpcErr},
{name: "s2", state: transactions.VoteCommitted},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedUpdated: []string{"s1"},
expectedOutdated: []string{"s2"},
expectedMetrics: map[string]int{
"node-error-status": 1,
"updated": 1,
},
},
{
desc: "partial success",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
},
secondaries: []node{
{name: "s1", state: transactions.VoteFailed},
{name: "s2", state: transactions.VoteCommitted},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedUpdated: []string{"s2"},
expectedOutdated: []string{"s1"},
expectedMetrics: map[string]int{
"node-not-committed": 1,
"updated": 1,
},
},
{
desc: "failure with (impossible) secondary success",
primary: node{
name: "primary",
state: transactions.VoteFailed,
},
secondaries: []node{
{name: "s1", state: transactions.VoteFailed},
{name: "s2", state: transactions.VoteCommitted},
},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1", "s2"},
expectedMetrics: map[string]int{
"primary-not-committed": 2,
},
},
{
desc: "failure with no primary votes",
primary: node{
name: "primary",
state: transactions.VoteFailed,
},
secondaries: []node{
{name: "s1", state: transactions.VoteFailed},
{name: "s2", state: transactions.VoteCommitted},
},
didVote: map[string]bool{
"s1": true,
"s2": true,
},
subtransactions: 1,
},
{
desc: "multiple nodes without subtransactions",
primary: node{
name: "primary",
state: transactions.VoteFailed,
},
secondaries: []node{
{name: "s1", state: transactions.VoteFailed},
{name: "s2", state: transactions.VoteCommitted},
},
subtransactions: 0,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1", "s2"},
expectedMetrics: map[string]int{
"no-votes": 2,
},
},
{
desc: "multiple nodes with replica and partial failures",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
},
secondaries: []node{
{name: "s1", state: transactions.VoteFailed},
{name: "s2", state: transactions.VoteCommitted},
},
replicas: []string{"r1", "r2"},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1", "r1", "r2"},
expectedUpdated: []string{"s2"},
expectedMetrics: map[string]int{
"node-not-committed": 1,
"outdated": 2,
"updated": 1,
},
},
{
desc: "multiple nodes with replica and partial err",
primary: node{
name: "primary",
state: transactions.VoteCommitted,
},
secondaries: []node{
{name: "s1", state: transactions.VoteFailed},
{name: "s2", state: transactions.VoteCommitted, err: grpcErr},
},
replicas: []string{"r1", "r2"},
didVote: map[string]bool{
"primary": true,
},
subtransactions: 1,
expectedPrimaryDirtied: true,
expectedOutdated: []string{"s1", "s2", "r1", "r2"},
expectedMetrics: map[string]int{
"node-error-status": 1,
"node-not-committed": 1,
"outdated": 2,
},
},
} {
t.Run(tc.desc, func(t *testing.T) {
nodes := append(tc.secondaries, tc.primary)
voters := make([]transactions.Voter, len(nodes))
states := make(map[string]transactions.VoteResult)
nodeErrors := &nodeErrors{
errByNode: make(map[string]error),
}
for i, node := range nodes {
voters[i] = transactions.Voter{
Name: node.name,
Votes: 1,
}
states[node.name] = node.state
nodeErrors.errByNode[node.name] = node.err
}
transaction := mockTransaction{
nodeStates: states,
subtransactions: tc.subtransactions,
didVote: tc.didVote,
}
route := RepositoryMutatorRoute{
Primary: RouterNode{
Storage: tc.primary.name,
},
}
for _, secondary := range tc.secondaries {
route.Secondaries = append(route.Secondaries, RouterNode{
Storage: secondary.name,
})
}
route.ReplicationTargets = append(route.ReplicationTargets, tc.replicas...)
metric := prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "stub", Help: "help",
}, []string{"reason"})
primaryDirtied, updated, outdated := getUpdatedAndOutdatedSecondaries(ctx, testhelper.NewLogger(t), route, transaction, nodeErrors, metric)
require.Equal(t, tc.expectedPrimaryDirtied, primaryDirtied)
require.ElementsMatch(t, tc.expectedUpdated, updated)
require.ElementsMatch(t, tc.expectedOutdated, outdated)
expectedMetrics := "# HELP stub help\n# TYPE stub counter\n"
for metric, value := range tc.expectedMetrics {
expectedMetrics += fmt.Sprintf("stub{reason=\"%s\"} %d\n", metric, value)
}
testhelper.RequirePromMetrics(t, metric, expectedMetrics)
})
}
}
func TestNewRequestFinalizer_contextIsDisjointedFromTheRPC(t *testing.T) {
type ctxKey struct{}
parentDeadline := time.Now()
ctx := testhelper.Context(t)
ctx, cancel := context.WithDeadline(context.WithValue(ctx, ctxKey{}, "value"), parentDeadline)
defer cancel()
requireSuppressedCancellation := func(tb testing.TB, ctx context.Context) {
deadline, ok := ctx.Deadline()
require.True(tb, ok)
require.NotEqual(tb, parentDeadline, deadline)
require.Equal(tb, ctx.Value(ctxKey{}), "value")
require.Nil(tb, ctx.Err())
select {
case <-ctx.Done():
tb.Fatal("context should not be canceled if the parent is canceled")
default:
require.NotNil(tb, ctx.Done())
}
}
err := errors.New("error")
for _, tc := range []struct {
desc string
change datastore.ChangeType
errMsg string
enqueueError error
}{
{
desc: "increment generation receives suppressed cancellation",
change: datastore.UpdateRepo,
errMsg: "increment generation: error",
},
{
desc: "enqueue receives suppressed cancellation",
errMsg: "enqueue replication event: error",
enqueueError: err,
},
{
desc: "original context error is returned",
errMsg: context.DeadlineExceeded.Error(),
},
} {
t.Run(tc.desc, func(t *testing.T) {
require.EqualError(t,
NewCoordinator(
testhelper.NewLogger(t),
&datastore.MockReplicationEventQueue{
EnqueueFunc: func(ctx context.Context, _ datastore.ReplicationEvent) (datastore.ReplicationEvent, error) {
requireSuppressedCancellation(t, ctx)
return datastore.ReplicationEvent{}, tc.enqueueError
},
},
datastore.MockRepositoryStore{
IncrementGenerationFunc: func(ctx context.Context, _ int64, _ string, _ []string) error {
requireSuppressedCancellation(t, ctx)
return err
},
CreateRepositoryFunc: func(ctx context.Context, _ int64, _, _, _, _ string, _, _ []string, _, _ bool) error {
requireSuppressedCancellation(t, ctx)
return err
},
},
nil,
nil,
nil,
config.Config{},
nil,
).newRequestFinalizer(
ctx,
0,
"virtual storage",
&gitalypb.Repository{},
"replica-path",
"primary",
[]string{},
[]string{"secondary"},
tc.change,
datastore.Params{"RelativePath": "relative-path"},
"rpc-name",
"",
)(),
tc.errMsg,
)
})
}
}
func TestNewRequestFinalizer_enqueueErrorPropagation(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for _, tc := range []struct {
desc string
enqueueErr error
expectedErr error
}{
{
desc: "most errors are propagated back",
enqueueErr: errors.New("enqueue failed"),
expectedErr: fmt.Errorf("enqueue replication event: %w", errors.New("enqueue failed")),
},
{
desc: "replication event exists errors are ignored ",
enqueueErr: datastore.ReplicationEventExistsError{},
expectedErr: nil,
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
err := NewCoordinator(
testhelper.NewLogger(t),
&datastore.MockReplicationEventQueue{
EnqueueFunc: func(ctx context.Context, _ datastore.ReplicationEvent) (datastore.ReplicationEvent, error) {
return datastore.ReplicationEvent{}, tc.enqueueErr
},
},
datastore.MockRepositoryStore{},
nil,
nil,
nil,
config.Config{},
nil,
).newRequestFinalizer(ctx,
1,
"praefect",
&gitalypb.Repository{},
"replic-path",
"primary",
[]string{},
[]string{"secondary"},
datastore.UpdateRepo,
datastore.Params{"RelativePath": "relative-path"},
"rpc-name",
"",
)()
require.Equal(t, tc.expectedErr, err)
})
}
}
func TestStreamParametersContext(t *testing.T) {
// Because we're using NewFeatureFlag, they'll end up in the All array.
enabledFF := featureflag.NewFeatureFlag("default_enabled", "", "", true)
disabledFF := featureflag.NewFeatureFlag("default_disabled", "", "", false)
type expectedFlag struct {
flag featureflag.FeatureFlag
enabled bool
}
expectedFlags := func(overrides ...expectedFlag) []expectedFlag {
flagValues := map[featureflag.FeatureFlag]bool{}
for _, flag := range featureflag.DefinedFlags() {
flagValues[flag] = flag.OnByDefault
}
for _, override := range overrides {
flagValues[override.flag] = override.enabled
}
expectedFlags := make([]expectedFlag, 0, len(flagValues))
for flag, value := range flagValues {
expectedFlags = append(expectedFlags, expectedFlag{
flag: flag, enabled: value,
})
}
return expectedFlags
}
metadataForFlags := func(flags []expectedFlag) metadata.MD {
pairs := []string{}
for _, flag := range flags {
pairs = append(pairs, flag.flag.MetadataKey(), strconv.FormatBool(flag.enabled))
}
return metadata.Pairs(pairs...)
}
// We explicitly test context values, so we cannot use the testhelper context here given that it would contain
// unrelated data and thus change the system under test.
for _, tc := range []struct {
desc string
setupContext func() context.Context
expectedIncomingMD metadata.MD
expectedOutgoingMD metadata.MD
expectedFlags []expectedFlag
}{
{
desc: "no metadata",
setupContext: func() context.Context {
return context.Background()
},
expectedFlags: expectedFlags(),
expectedOutgoingMD: metadataForFlags(expectedFlags()),
},
{
desc: "with incoming metadata",
setupContext: func() context.Context {
ctx := context.Background()
ctx = metadata.NewIncomingContext(ctx, metadata.Pairs("key", "value"))
return ctx
},
expectedIncomingMD: metadata.Pairs("key", "value"),
expectedOutgoingMD: metadata.Join(
metadata.Pairs("key", "value"),
metadataForFlags(expectedFlags()),
),
expectedFlags: expectedFlags(),
},
{
desc: "with outgoing metadata",
setupContext: func() context.Context {
ctx := context.Background()
ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("key", "value"))
return ctx
},
expectedOutgoingMD: metadata.Join(
metadata.Pairs("key", "value"),
metadataForFlags(expectedFlags()),
),
expectedFlags: expectedFlags(),
},
{
desc: "with incoming and outgoing metadata",
setupContext: func() context.Context {
ctx := context.Background()
ctx = metadata.NewIncomingContext(ctx, metadata.Pairs("incoming", "value"))
ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("outgoing", "value"))
return ctx
},
// This behaviour is quite subtle: in the previous test case where we only
// have outgoing metadata, we retain it. But in case we have both incoming
// and outgoing we'd discard the outgoing metadata altogether. It is
// debatable whether this is a bug or feature, so I'll just document this
// weird edge case here for now.
expectedIncomingMD: metadata.Pairs("incoming", "value"),
expectedOutgoingMD: metadata.Join(
metadata.Pairs("incoming", "value"),
metadataForFlags(expectedFlags()),
),
expectedFlags: expectedFlags(),
},
{
desc: "with flags set to their default values",
setupContext: func() context.Context {
ctx := context.Background()
ctx = featureflag.IncomingCtxWithFeatureFlag(ctx, enabledFF, true)
ctx = featureflag.IncomingCtxWithFeatureFlag(ctx, disabledFF, false)
return ctx
},
expectedIncomingMD: metadata.Pairs(
enabledFF.MetadataKey(), "true",
disabledFF.MetadataKey(), "false",
),
expectedOutgoingMD: metadata.Join(
metadataForFlags(expectedFlags()),
),
expectedFlags: expectedFlags(),
},
{
desc: "with flags set to their reverse default values",
setupContext: func() context.Context {
ctx := context.Background()
ctx = featureflag.IncomingCtxWithFeatureFlag(ctx, enabledFF, false)
ctx = featureflag.IncomingCtxWithFeatureFlag(ctx, disabledFF, true)
return ctx
},
expectedIncomingMD: metadata.Pairs(
enabledFF.MetadataKey(), "false",
disabledFF.MetadataKey(), "true",
),
expectedOutgoingMD: metadata.Join(
metadataForFlags(expectedFlags(
expectedFlag{flag: enabledFF, enabled: false},
expectedFlag{flag: disabledFF, enabled: true},
)),
),
expectedFlags: expectedFlags(
expectedFlag{flag: enabledFF, enabled: false},
expectedFlag{flag: disabledFF, enabled: true},
),
},
{
desc: "mixed flags and metadata",
setupContext: func() context.Context {
ctx := context.Background()
ctx = metadata.NewIncomingContext(ctx, metadata.Pairs(
disabledFF.MetadataKey(), "true",
"incoming", "value"),
)
return ctx
},
expectedIncomingMD: metadata.Pairs(
disabledFF.MetadataKey(), "true",
"incoming", "value",
),
expectedOutgoingMD: metadata.Join(
metadataForFlags(expectedFlags(
expectedFlag{flag: disabledFF, enabled: true},
)),
metadata.Pairs(
"incoming", "value",
),
),
expectedFlags: expectedFlags(
expectedFlag{flag: disabledFF, enabled: true},
),
},
} {
t.Run(tc.desc, func(t *testing.T) {
ctx := streamParametersContext(tc.setupContext())
incomingMD, ok := metadata.FromIncomingContext(ctx)
if tc.expectedIncomingMD == nil {
require.False(t, ok)
} else {
require.True(t, ok)
}
require.Equal(t, tc.expectedIncomingMD, incomingMD)
outgoingMD, ok := metadata.FromOutgoingContext(ctx)
if tc.expectedOutgoingMD == nil {
require.False(t, ok)
} else {
require.True(t, ok)
}
require.Equal(t, tc.expectedOutgoingMD, outgoingMD)
incomingCtx := gitaly_metadata.OutgoingToIncoming(ctx)
for _, expectedFlag := range tc.expectedFlags {
require.Equal(t, expectedFlag.enabled, expectedFlag.flag.IsEnabled(incomingCtx))
}
})
}
}