Skip to content

Git designed for AI agents.

12.5 KB
Raw Blame
package datastore

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"testing"
	"time"

	"github.com/feder-cr/git-ai/internal/log"
	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/feder-cr/git-ai/internal/testhelper/testdb"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestRepositoryStoreCollector(t *testing.T) {
	t.Parallel()
	ctx := testhelper.Context(t)

	type replicas map[string]struct {
		generation int
		assigned   bool
	}

	type repositories []struct {
		deleted      bool
		relativePath string
		replicas     replicas
	}

	db := testdb.New(t)

	for _, tc := range []struct {
		desc         string
		healthyNodes []string
		repositories repositories
		timeout      bool
		count        int
		error        error
	}{
		{
			desc: "no repositories",
		},
		{
			desc: "deleted repositories are not considered unavailable",
			repositories: repositories{
				{
					deleted:      true,
					relativePath: "repository-1",
					replicas: replicas{
						"storage-1": {generation: 0},
					},
				},
			},
		},
		{
			desc: "repositories without any healthy replicas are counted",
			repositories: repositories{
				{
					relativePath: "repository-1",
					replicas: replicas{
						"storage-1": {generation: 0},
					},
				},
			},
			count: 1,
		},
		{
			desc:         "repositories with healthy replicas are not counted",
			healthyNodes: []string{"storage-1"},
			repositories: repositories{
				{
					relativePath: "repository-1",
					replicas: replicas{
						"storage-1": {generation: 0},
					},
				},
			},
		},
		{
			desc:         "repositories with only outdated healthy replicas are counted",
			healthyNodes: []string{"storage-1"},
			repositories: repositories{
				{
					relativePath: "repository-1",
					replicas: replicas{
						"storage-1": {generation: 0},
						"storage-2": {generation: 1},
					},
				},
			},
			count: 1,
		},
		{
			desc:         "repositories with unassigned fully up to date healthy replicas are not counted",
			healthyNodes: []string{"storage-2"},
			repositories: repositories{
				{
					relativePath: "repository-1",
					replicas: replicas{
						"storage-1": {generation: 1, assigned: true},
						"storage-2": {generation: 1},
					},
				},
			},
		},
		{
			desc:         "repositories with unassigned, outdated replicas is not unavailable",
			healthyNodes: []string{"storage-1"},
			repositories: repositories{
				{
					relativePath: "repository-1",
					replicas: replicas{
						"storage-1": {generation: 1, assigned: true},
						"storage-2": {generation: 0},
					},
				},
			},
		},
		{
			desc:         "multiple unavailable repositories are counted correctly",
			healthyNodes: []string{"storage-2"},
			repositories: repositories{
				{
					relativePath: "repository-1",
					replicas: replicas{
						"storage-1": {generation: 1},
						"storage-2": {generation: 0},
					},
				},
				{
					relativePath: "repository-2",
					replicas: replicas{
						"storage-1": {generation: 1},
						"storage-2": {generation: 0},
					},
				},
			},
			count: 2,
		},
		{
			desc:    "query timeout",
			timeout: true,
			error:   fmt.Errorf("query: %w", context.DeadlineExceeded),
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			tx := db.Begin(t)
			defer tx.Rollback(t)

			testdb.SetHealthyNodes(t, ctx, tx, map[string]map[string][]string{
				"praefect-0": {"virtual-storage-1": tc.healthyNodes},
			})

			for _, repository := range tc.repositories {

				rs := NewPostgresRepositoryStore(tx, nil)

				var repositoryID int64
				for storage, replica := range repository.replicas {
					if repositoryID == 0 {
						const virtualStorage = "virtual-storage-1"

						var err error
						repositoryID, err = rs.ReserveRepositoryID(ctx, virtualStorage, repository.relativePath)
						require.NoError(t, err)

						require.NoError(t, rs.CreateRepository(ctx, repositoryID, virtualStorage, repository.relativePath, repository.relativePath, storage, nil, nil, false, false))
					}

					if replica.assigned {
						_, err := tx.ExecContext(ctx, `
							INSERT INTO repository_assignments (repository_id, virtual_storage, relative_path, storage)
							VALUES ($1, 'virtual-storage-1', $2, $3)
						`, repositoryID, repository.relativePath, storage,
						)
						require.NoError(t, err)
					}

					require.NoError(t, rs.SetGeneration(ctx, repositoryID, storage, repository.relativePath, replica.generation))
				}

				if repository.deleted {
					_, err := tx.ExecContext(ctx, `
						DELETE FROM repositories WHERE repository_id = $1
					`, repositoryID,
					)
					require.NoError(t, err)
				}
			}

			timeout := time.Hour
			if tc.timeout {
				timeout = 0
			}

			logger := testhelper.NewLogger(t)
			hook := testhelper.AddLoggerHook(logger)

			c := NewRepositoryStoreCollector(logger, []string{"virtual-storage-1", "virtual-storage-2"}, tx, timeout)
			err := testhelper.ComparePromMetrics(t, c, fmt.Sprintf(`
# HELP gitaly_praefect_unavailable_repositories Number of repositories that have no healthy, up to date replicas.
# TYPE gitaly_praefect_unavailable_repositories gauge
gitaly_praefect_unavailable_repositories{virtual_storage="virtual-storage-1"} %d
gitaly_praefect_unavailable_repositories{virtual_storage="virtual-storage-2"} 0
			`, tc.count))

			if tc.error != nil {
				require.Equal(t, "failed collecting unavailable repository count metric", hook.AllEntries()[0].Message)
				require.Equal(t, log.Fields{"error": tc.error, "component": "RepositoryStoreCollector"}, hook.AllEntries()[0].Data)
				return
			}

			require.NoError(t, err)
		})
	}
}

type checkIfQueriedDB struct {
	queried bool
}

func (c *checkIfQueriedDB) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) {
	c.queried = true
	return nil, errors.New("QueryContext should not be called")
}

func (c *checkIfQueriedDB) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
	c.queried = true
	return &sql.Row{}
}

func (c *checkIfQueriedDB) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
	c.queried = true
	return nil, errors.New("ExecContext should not be called")
}

func TestRepositoryStoreCollector_CollectNotCalledOnRegister(t *testing.T) {
	logger := testhelper.NewLogger(t)

	var db checkIfQueriedDB
	c := NewRepositoryStoreCollector(logger, []string{"virtual-storage-1", "virtual-storage-2"}, &db, 2*time.Second)
	registry := prometheus.NewRegistry()
	registry.MustRegister(c)

	assert.False(t, db.queried)
}

func TestRepositoryStoreCollector_ReplicationQueueDepth(t *testing.T) {
	db := testdb.New(t)
	ctx := testhelper.Context(t)

	log := testhelper.SharedLogger(t)

	storageNames := map[string][]string{
		"praefect-0": {"storage-0", "storage-1", "storage-2"},
		"praefect-1": {"storage-3", "storage-4", "storage-5"},
	}
	queue := NewPostgresReplicationEventQueue(db.DB)

	readyJobs := 5
	for virtualStorage, nodes := range storageNames {
		for i := 0; i < readyJobs; i++ {
			_, err := queue.Enqueue(ctx, ReplicationEvent{
				Job: ReplicationJob{
					Change:            UpdateRepo,
					RelativePath:      fmt.Sprintf("/project/path-%d", i),
					TargetNodeStorage: nodes[1],
					SourceNodeStorage: nodes[0],
					VirtualStorage:    virtualStorage,
					RepositoryID:      int64(i),
					Params:            nil,
				},
			})
			require.NoError(t, err)
		}
	}

	collector := NewQueueDepthCollector(log, db, 1*time.Hour)

	testhelper.RequirePromMetrics(t, collector, fmt.Sprintf(`
# HELP gitaly_praefect_replication_queue_depth Number of jobs in the replication queue
# TYPE gitaly_praefect_replication_queue_depth gauge
gitaly_praefect_replication_queue_depth{state="ready",target_node="storage-1",virtual_storage="praefect-0"} %d
gitaly_praefect_replication_queue_depth{state="ready",target_node="storage-4",virtual_storage="praefect-1"} %d
`, readyJobs, readyJobs))

	var eventIDs []uint64
	events, err := queue.Dequeue(ctx, "praefect-0", "storage-1", 1)
	require.NoError(t, err)
	require.Len(t, events, 1)
	eventIDs = append(eventIDs, events[0].ID)

	events, err = queue.Dequeue(ctx, "praefect-1", "storage-4", 1)
	require.NoError(t, err)
	require.Len(t, events, 1)
	eventIDs = append(eventIDs, events[0].ID)

	testhelper.RequirePromMetrics(t, collector, fmt.Sprintf(`
# HELP gitaly_praefect_replication_queue_depth Number of jobs in the replication queue
# TYPE gitaly_praefect_replication_queue_depth gauge
gitaly_praefect_replication_queue_depth{state="in_progress",target_node="storage-1",virtual_storage="praefect-0"} %d
gitaly_praefect_replication_queue_depth{state="in_progress",target_node="storage-4",virtual_storage="praefect-1"} %d
gitaly_praefect_replication_queue_depth{state="ready",target_node="storage-1",virtual_storage="praefect-0"} %d
gitaly_praefect_replication_queue_depth{state="ready",target_node="storage-4",virtual_storage="praefect-1"} %d
`, 1, 1, readyJobs-1, readyJobs-1))

	_, err = queue.Acknowledge(ctx, JobStateFailed, eventIDs)
	require.NoError(t, err)

	testhelper.RequirePromMetrics(t, collector, fmt.Sprintf(`
# HELP gitaly_praefect_replication_queue_depth Number of jobs in the replication queue
# TYPE gitaly_praefect_replication_queue_depth gauge
gitaly_praefect_replication_queue_depth{state="failed",target_node="storage-1",virtual_storage="praefect-0"} %d
gitaly_praefect_replication_queue_depth{state="failed",target_node="storage-4",virtual_storage="praefect-1"} %d
gitaly_praefect_replication_queue_depth{state="ready",target_node="storage-1",virtual_storage="praefect-0"} %d
gitaly_praefect_replication_queue_depth{state="ready",target_node="storage-4",virtual_storage="praefect-1"} %d
`, 1, 1, readyJobs-1, readyJobs-1))
}

func TestVerificationQueueDepthCollector(t *testing.T) {
	ctx := testhelper.Context(t)

	tx := testdb.New(t).Begin(t)
	defer tx.Rollback(t)

	rs := NewPostgresRepositoryStore(tx, nil)
	require.NoError(t,
		rs.CreateRepository(ctx, 1, "virtual-storage-1", "relative-path-1", "replica-path-1", "gitaly-1", []string{"gitaly-2", "gitaly-3"}, nil, true, false),
	)
	require.NoError(t,
		rs.CreateRepository(ctx, 2, "virtual-storage-1", "relative-path-2", "replica-path-2", "gitaly-1", []string{"gitaly-2", "gitaly-3"}, nil, true, false),
	)
	require.NoError(t,
		rs.CreateRepository(ctx, 3, "virtual-storage-2", "relative-path-1", "replica-path-3", "gitaly-1", []string{"gitaly-2", "gitaly-3"}, nil, true, false),
	)

	_, err := tx.ExecContext(ctx, `
UPDATE storage_repositories
SET verified_at = CASE
	WHEN storage = 'gitaly-2' THEN now() - '30 seconds'::interval
	ELSE now() - '30 seconds'::interval - '1 microsecond'::interval
END
WHERE virtual_storage = 'virtual-storage-1' AND storage != 'gitaly-1'
	`)
	require.NoError(t, err)

	logger := testhelper.NewLogger(t)
	hook := testhelper.AddLoggerHook(logger)
	testhelper.RequirePromMetrics(
		t,
		NewVerificationQueueDepthCollector(logger, tx, time.Minute, 30*time.Second, map[string][]string{
			"virtual-storage-1": {"gitaly-1", "gitaly-2", "gitaly-3"},
			"virtual-storage-2": {"gitaly-1", "gitaly-2", "gitaly-3"},
		}),
		`
# HELP gitaly_praefect_verification_queue_depth Number of replicas pending verification.
# TYPE gitaly_praefect_verification_queue_depth gauge
gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-1",virtual_storage="virtual-storage-1"} 0
gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-1",virtual_storage="virtual-storage-2"} 0
gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-2",virtual_storage="virtual-storage-1"} 0
gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-2",virtual_storage="virtual-storage-2"} 0
gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-3",virtual_storage="virtual-storage-1"} 2
gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-3",virtual_storage="virtual-storage-2"} 0
gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-1",virtual_storage="virtual-storage-1"} 2
gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-1",virtual_storage="virtual-storage-2"} 1
gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-2",virtual_storage="virtual-storage-1"} 0
gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-2",virtual_storage="virtual-storage-2"} 1
gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-3",virtual_storage="virtual-storage-1"} 0
gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-3",virtual_storage="virtual-storage-2"} 1
		`,
	)
	require.Empty(t, hook.AllEntries())
}