Skip to content

Git designed for AI agents.

19.9 KB
Raw Blame
package limiter

import (
	"testing"
	"time"

	promconfig "github.com/feder-cr/git-ai/internal/gitaly/config/prometheus"
	"github.com/feder-cr/git-ai/internal/log"
	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/stretchr/testify/require"
)

func TestNewPerRPCPromMonitor(t *testing.T) {
	system := "gitaly"
	fullMethod := "fullMethod"
	createNewMonitor := func(authenticated string) *PromMonitor {
		acquiringSecondsVec := prometheus.NewHistogramVec(
			prometheus.HistogramOpts{
				Name:    "acquiring_seconds",
				Help:    "seconds to acquire",
				Buckets: promconfig.DefaultConfig().GRPCLatencyBuckets,
			},
			[]string{"system", "grpc_service", "grpc_method", "authenticated"},
		)
		inProgressMetric := prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Name: "in_progress",
				Help: "requests in progress",
			},
			[]string{"system", "grpc_service", "grpc_method", "authenticated"},
		)
		queuedMetric := prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Name: "queued",
				Help: "number of queued requests",
			},
			[]string{"system", "grpc_service", "grpc_method", "authenticated"},
		)
		requestsDroppedMetric := prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Name: "dropped",
				Help: "number of dropped requests",
			},
			[]string{
				"system",
				"grpc_service",
				"grpc_method",
				"reason",
				"authenticated",
			},
		)
		return NewPerRPCPromMonitor(
			system,
			fullMethod,
			queuedMetric,
			inProgressMetric,
			acquiringSecondsVec,
			requestsDroppedMetric,
			authenticated,
		)
	}

	t.Run("request is dequeued successfully", func(t *testing.T) {
		rpcMonitor := createNewMonitor(LabelAuthenticated)
		ctx := log.InitContextCustomFields(testhelper.Context(t))

		rpcMonitor.Queued(ctx, fullMethod, 5)
		rpcMonitor.Enter(ctx, 10, time.Second)

		expectedMetrics := `# HELP acquiring_seconds seconds to acquire
# TYPE acquiring_seconds histogram
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.001"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.005"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.025"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.1"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.5"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1"} 1
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="10"} 1
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="30"} 1
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="60"} 1
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="300"} 1
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1500"} 1
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="+Inf"} 1
acquiring_seconds_sum{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1
acquiring_seconds_count{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1
# HELP in_progress requests in progress
# TYPE in_progress gauge
in_progress{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1
# HELP queued number of queued requests
# TYPE queued gauge
queued{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1

`
		testhelper.RequirePromMetrics(t, rpcMonitor, expectedMetrics)

		stats := log.CustomFieldsFromContext(ctx)
		require.NotNil(t, stats)
		require.Equal(t, log.Fields{
			"limit.limiting_type":            TypePerRPC,
			"limit.limiting_key":             fullMethod,
			"limit.concurrency_queue_ms":     int64(1000),
			"limit.concurrency_queue_length": 5,
			"limit.concurrency_in_progress":  10,
		}, stats.Fields())

		// After the request exists, in_progress and queued gauge decrease
		rpcMonitor.Exit(ctx)
		rpcMonitor.Dequeued(ctx)
		expectedMetrics = `# HELP acquiring_seconds seconds to acquire
# HELP in_progress requests in progress
# TYPE in_progress gauge
in_progress{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
# HELP queued number of queued requests
# TYPE queued gauge
queued{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0

`
		testhelper.RequirePromMetrics(t, rpcMonitor, expectedMetrics)
	})

	t.Run("request is dropped after queueing", func(t *testing.T) {
		rpcMonitor := createNewMonitor(LabelAuthenticated)
		ctx := log.InitContextCustomFields(testhelper.Context(t))

		rpcMonitor.Queued(ctx, fullMethod, 5)
		rpcMonitor.Dropped(ctx, fullMethod, 5, 10, time.Second, "load")

		expectedMetrics := `# HELP acquiring_seconds seconds to acquire
# TYPE acquiring_seconds histogram
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.001"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.005"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.025"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.1"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.5"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="10"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="30"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="60"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="300"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1500"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="+Inf"} 0
acquiring_seconds_sum{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
acquiring_seconds_count{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
# HELP dropped number of dropped requests
# TYPE dropped counter
dropped{authenticated="true",grpc_method="unknown",grpc_service="unknown",reason="load",system="gitaly"} 1
# HELP in_progress requests in progress
# TYPE in_progress gauge
in_progress{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
# HELP queued number of queued requests
# TYPE queued gauge
queued{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1

`
		testhelper.RequirePromMetrics(t, rpcMonitor, expectedMetrics)
		stats := log.CustomFieldsFromContext(ctx)
		require.NotNil(t, stats)
		require.Equal(t, log.Fields{
			"limit.limiting_type":            TypePerRPC,
			"limit.limiting_key":             fullMethod,
			"limit.concurrency_queue_ms":     int64(1000),
			"limit.concurrency_queue_length": 5,
			"limit.concurrency_in_progress":  10,
			"limit.concurrency_dropped":      "load",
			"limit.authenticated":            "true",
		}, stats.Fields())
	})

	t.Run("request is dropped before queueing", func(t *testing.T) {
		rpcMonitor := createNewMonitor(LabelAuthenticated)
		ctx := log.InitContextCustomFields(testhelper.Context(t))
		rpcMonitor.Dropped(ctx, fullMethod, 5, 10, time.Second, "load")

		expectedMetrics := `# HELP acquiring_seconds seconds to acquire
# TYPE acquiring_seconds histogram
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.001"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.005"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.025"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.1"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.5"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="10"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="30"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="60"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="300"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1500"} 0
acquiring_seconds_bucket{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly",le="+Inf"} 0
acquiring_seconds_sum{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
acquiring_seconds_count{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
# HELP dropped number of dropped requests
# TYPE dropped counter
dropped{authenticated="true",grpc_method="unknown",grpc_service="unknown",reason="load",system="gitaly"} 1
# HELP in_progress requests in progress
# TYPE in_progress gauge
in_progress{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
# HELP queued number of queued requests
# TYPE queued gauge
queued{authenticated="true",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0

`
		testhelper.RequirePromMetrics(t, rpcMonitor, expectedMetrics)

		stats := log.CustomFieldsFromContext(ctx)
		require.NotNil(t, stats)
		require.Equal(t, log.Fields{
			"limit.limiting_type":            TypePerRPC,
			"limit.limiting_key":             fullMethod,
			"limit.concurrency_queue_ms":     int64(1000),
			"limit.concurrency_queue_length": 5,
			"limit.concurrency_in_progress":  10,
			"limit.concurrency_dropped":      "load",
			"limit.authenticated":            "true",
		}, stats.Fields())
	})

	t.Run("unauthenticated monitor emits authenticated=false label", func(t *testing.T) {
		rpcMonitor := createNewMonitor(LabelUnauthenticated)
		ctx := log.InitContextCustomFields(testhelper.Context(t))

		rpcMonitor.Queued(ctx, fullMethod, 5)
		rpcMonitor.Enter(ctx, 10, time.Second)

		expectedMetrics := `# HELP in_progress requests in progress
# TYPE in_progress gauge
in_progress{authenticated="false",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1
# HELP queued number of queued requests
# TYPE queued gauge
queued{authenticated="false",grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1
`
		testhelper.RequirePromMetrics(t, rpcMonitor, expectedMetrics)

		// Dropping a request records the unauthenticated value in the log fields too.
		rpcMonitor.Dropped(ctx, fullMethod, 5, 10, time.Second, "load")
		stats := log.CustomFieldsFromContext(ctx)
		require.NotNil(t, stats)
		require.Equal(t, "false", stats.Fields()["limit.authenticated"])
	})
}

func TestNewPackObjectsConcurrencyMonitor(t *testing.T) {
	t.Run("request is dequeued successfully", func(t *testing.T) {
		ctx := log.InitContextCustomFields(testhelper.Context(t))
		packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor(
			promconfig.DefaultConfig().GRPCLatencyBuckets,
			"authenticated",
		)

		packObjectsConcurrencyMonitor.Queued(ctx, "1234", 5)
		packObjectsConcurrencyMonitor.Enter(ctx, 10, time.Second)

		expectedMetrics := `# HELP gitaly_pack_objects_acquiring_seconds Histogram of time calls are rate limited (in seconds)
# TYPE gitaly_pack_objects_acquiring_seconds histogram
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.001"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.005"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.025"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.1"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.5"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="1"} 1
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="10"} 1
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="30"} 1
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="60"} 1
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="300"} 1
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="1500"} 1
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="+Inf"} 1
gitaly_pack_objects_acquiring_seconds_sum{auth="authenticated"} 1
gitaly_pack_objects_acquiring_seconds_count{auth="authenticated"} 1
# HELP gitaly_pack_objects_in_progress Gauge of number of concurrent in-progress calls
# TYPE gitaly_pack_objects_in_progress gauge
gitaly_pack_objects_in_progress{auth="authenticated"} 1
# HELP gitaly_pack_objects_queued Gauge of number of queued calls
# TYPE gitaly_pack_objects_queued gauge
gitaly_pack_objects_queued{auth="authenticated"} 1

`
		testhelper.RequirePromMetrics(t, packObjectsConcurrencyMonitor, expectedMetrics)

		stats := log.CustomFieldsFromContext(ctx)
		require.NotNil(t, stats)
		require.Equal(t, log.Fields{
			"limit.limiting_type":            TypePackObjects,
			"limit.limiting_key":             "1234",
			"limit.concurrency_queue_ms":     int64(1000),
			"limit.concurrency_queue_length": 5,
			"limit.concurrency_in_progress":  10,
		}, stats.Fields())

		// After the request exists, in_progress and queued gauge decrease
		packObjectsConcurrencyMonitor.Exit(ctx)
		packObjectsConcurrencyMonitor.Dequeued(ctx)
		expectedMetrics = `# HELP acquiring_seconds seconds to acquire
# HELP gitaly_pack_objects_in_progress Gauge of number of concurrent in-progress calls
# TYPE gitaly_pack_objects_in_progress gauge
gitaly_pack_objects_in_progress{auth="authenticated"} 0
# HELP gitaly_pack_objects_queued Gauge of number of queued calls
# TYPE gitaly_pack_objects_queued gauge
gitaly_pack_objects_queued{auth="authenticated"} 0

`
		testhelper.RequirePromMetrics(t, packObjectsConcurrencyMonitor, expectedMetrics)
	})

	t.Run("request is dropped after queueing", func(t *testing.T) {
		ctx := log.InitContextCustomFields(testhelper.Context(t))
		packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor(
			promconfig.DefaultConfig().GRPCLatencyBuckets,
			"authenticated",
		)

		packObjectsConcurrencyMonitor.Queued(ctx, "1234", 5)
		packObjectsConcurrencyMonitor.Dropped(ctx, "1234", 5, 10, time.Second, "load")

		expectedMetrics := `# HELP gitaly_pack_objects_acquiring_seconds Histogram of time calls are rate limited (in seconds)
# TYPE gitaly_pack_objects_acquiring_seconds histogram
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.001"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.005"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.025"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.1"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.5"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="1"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="10"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="30"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="60"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="300"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="1500"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="+Inf"} 0
gitaly_pack_objects_acquiring_seconds_sum{auth="authenticated"} 0
gitaly_pack_objects_acquiring_seconds_count{auth="authenticated"} 0
# HELP gitaly_pack_objects_dropped_total Number of requests dropped from the queue
# TYPE gitaly_pack_objects_dropped_total counter
gitaly_pack_objects_dropped_total{auth="authenticated",reason="load"} 1
# HELP gitaly_pack_objects_in_progress Gauge of number of concurrent in-progress calls
# TYPE gitaly_pack_objects_in_progress gauge
gitaly_pack_objects_in_progress{auth="authenticated"} 0
# HELP gitaly_pack_objects_queued Gauge of number of queued calls
# TYPE gitaly_pack_objects_queued gauge
gitaly_pack_objects_queued{auth="authenticated"} 1

`
		testhelper.RequirePromMetrics(t, packObjectsConcurrencyMonitor, expectedMetrics)

		stats := log.CustomFieldsFromContext(ctx)
		require.NotNil(t, stats)
		require.Equal(t, log.Fields{
			"limit.limiting_type":            TypePackObjects,
			"limit.limiting_key":             "1234",
			"limit.concurrency_queue_ms":     int64(1000),
			"limit.concurrency_queue_length": 5,
			"limit.concurrency_in_progress":  10,
			"limit.concurrency_dropped":      "load",
		}, stats.Fields())
	})

	t.Run("request is dropped before queueing", func(t *testing.T) {
		ctx := log.InitContextCustomFields(testhelper.Context(t))
		packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor(
			promconfig.DefaultConfig().GRPCLatencyBuckets,
			"authenticated",
		)

		packObjectsConcurrencyMonitor.Dropped(ctx, "1234", 5, 10, time.Second, "load")

		expectedMetrics := `# HELP gitaly_pack_objects_acquiring_seconds Histogram of time calls are rate limited (in seconds)
# TYPE gitaly_pack_objects_acquiring_seconds histogram
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.001"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.005"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.025"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.1"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="0.5"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="1"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="10"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="30"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="60"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="300"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="1500"} 0
gitaly_pack_objects_acquiring_seconds_bucket{auth="authenticated",le="+Inf"} 0
gitaly_pack_objects_acquiring_seconds_sum{auth="authenticated"} 0
gitaly_pack_objects_acquiring_seconds_count{auth="authenticated"} 0
# HELP gitaly_pack_objects_dropped_total Number of requests dropped from the queue
# TYPE gitaly_pack_objects_dropped_total counter
gitaly_pack_objects_dropped_total{auth="authenticated",reason="load"} 1
# HELP gitaly_pack_objects_in_progress Gauge of number of concurrent in-progress calls
# TYPE gitaly_pack_objects_in_progress gauge
gitaly_pack_objects_in_progress{auth="authenticated"} 0
# HELP gitaly_pack_objects_queued Gauge of number of queued calls
# TYPE gitaly_pack_objects_queued gauge
gitaly_pack_objects_queued{auth="authenticated"} 0

`
		testhelper.RequirePromMetrics(t, packObjectsConcurrencyMonitor, expectedMetrics)

		stats := log.CustomFieldsFromContext(ctx)
		require.NotNil(t, stats)
		require.Equal(t, log.Fields{
			"limit.limiting_type":            TypePackObjects,
			"limit.limiting_key":             "1234",
			"limit.concurrency_queue_ms":     int64(1000),
			"limit.concurrency_queue_length": 5,
			"limit.concurrency_in_progress":  10,
			"limit.concurrency_dropped":      "load",
		}, stats.Fields())
	})
}