Skip to content

Git designed for AI agents.

40.6 KB
Raw Blame
package datastore

import (
	"context"
	"sync"
	"testing"
	"time"

	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/feder-cr/git-ai/internal/testhelper/testdb"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestPostgresReplicationEventQueue_DeleteReplicaUniqueIndex(t *testing.T) {
	t.Parallel()
	db := testdb.New(t)
	for _, tc := range []struct {
		desc        string
		existingJob *ReplicationEvent
		succeeds    bool
	}{
		{
			desc:     "allowed when no events",
			succeeds: true,
		},
		{
			desc: "allowed if existing completed job",
			existingJob: &ReplicationEvent{
				State: JobStateCompleted,
				Job: ReplicationJob{
					Change:         DeleteReplica,
					VirtualStorage: "praefect",
					RelativePath:   "relative-path",
				},
			},
			succeeds: true,
		},
		{
			desc: "allowed if existing dead job",
			existingJob: &ReplicationEvent{
				State: JobStateDead,
				Job: ReplicationJob{
					Change:         DeleteReplica,
					VirtualStorage: "praefect",
					RelativePath:   "relative-path",
				},
			},
			succeeds: true,
		},
		{
			desc: "allowed if existing different virtual storage",
			existingJob: &ReplicationEvent{
				State: JobStateReady,
				Job: ReplicationJob{
					Change:         DeleteReplica,
					VirtualStorage: "wrong-virtual-storage",
					RelativePath:   "relative-path",
				},
			},
			succeeds: true,
		},
		{
			desc: "allowed if existing different relative path",
			existingJob: &ReplicationEvent{
				State: JobStateReady,
				Job: ReplicationJob{
					Change:         DeleteReplica,
					VirtualStorage: "praefect",
					RelativePath:   "wrong-relative-path",
				},
			},
			succeeds: true,
		},
		{
			desc: "not allowed if existing ready job",
			existingJob: &ReplicationEvent{
				State: JobStateReady,
				Job: ReplicationJob{
					Change:         DeleteReplica,
					VirtualStorage: "praefect",
					RelativePath:   "relative-path",
				},
			},
		},
		{
			desc: "not allowed if existing in_progress job",
			existingJob: &ReplicationEvent{
				State: JobStateInProgress,
				Job: ReplicationJob{
					Change:         DeleteReplica,
					VirtualStorage: "praefect",
					RelativePath:   "relative-path",
				},
			},
		},
		{
			desc: "not allowed if existing failed job",
			existingJob: &ReplicationEvent{
				State: JobStateFailed,
				Job: ReplicationJob{
					Change:         DeleteReplica,
					VirtualStorage: "praefect",
					RelativePath:   "relative-path",
				},
			},
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			db.TruncateAll(t)
			ctx := testhelper.Context(t)

			if tc.existingJob != nil {
				_, err := db.ExecContext(ctx, `
					INSERT INTO replication_queue (state, job)
					VALUES ($1, $2)
				`, tc.existingJob.State, tc.existingJob.Job)
				require.NoError(t, err)
			}

			_, err := NewPostgresReplicationEventQueue(db).Enqueue(ctx, ReplicationEvent{
				State: JobStateReady,
				Job: ReplicationJob{
					Change:            DeleteReplica,
					VirtualStorage:    "praefect",
					RelativePath:      "relative-path",
					TargetNodeStorage: "gitaly-1",
				},
			})

			if tc.succeeds {
				require.NoError(t, err)
				return
			}

			require.EqualError(t, err, `query: ERROR: duplicate key value violates unique constraint "delete_replica_unique_index" (SQLSTATE 23505)`)
		})
	}
}

func TestPostgresReplicationEventQueue_Enqueue(t *testing.T) {
	t.Parallel()
	db := testdb.New(t)
	ctx := testhelper.Context(t)

	queue := PostgresReplicationEventQueue{db.DB}

	eventType := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	actualEvent, err := queue.Enqueue(ctx, eventType) // initial event
	require.NoError(t, err)
	actualEvent.CreatedAt = time.Time{} // we need to setup it to default because it is not possible to get it beforehand for expected

	expLock := LockRow{ID: "praefect|gitaly-1|/project/path-1", Acquired: false}

	expEvent := ReplicationEvent{
		ID:      1,
		State:   JobStateReady,
		Attempt: 3,
		LockID:  "praefect|gitaly-1|/project/path-1",
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	require.Equal(t, expEvent, actualEvent)
	requireEvents(t, ctx, db, []ReplicationEvent{expEvent})
	requireLocks(t, ctx, db, []LockRow{expLock}) // expected a new lock for new event
	db.RequireRowsInTable(t, "replication_queue_job_lock", 0)
}

func TestPostgresReplicationEventQueue_DeleteReplicaInfiniteAttempts(t *testing.T) {
	t.Parallel()
	queue := NewPostgresReplicationEventQueue(testdb.New(t))
	ctx := testhelper.Context(t)

	actualEvent, err := queue.Enqueue(ctx, ReplicationEvent{
		Job: ReplicationJob{
			Change:            DeleteReplica,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			VirtualStorage:    "praefect",
		},
	})
	require.NoError(t, err)

	expectedEvent := ReplicationEvent{
		ID:      1,
		State:   JobStateReady,
		Attempt: 3,
		LockID:  "praefect|gitaly-1|/project/path-1",
		Job: ReplicationJob{
			Change:            DeleteReplica,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
		CreatedAt: actualEvent.CreatedAt,
	}

	require.Equal(t, expectedEvent, actualEvent)

	for i := 0; i < 2*actualEvent.Attempt; i++ {
		actualEvents, err := queue.Dequeue(ctx, "praefect", "gitaly-1", 9999)
		require.NoError(t, err)
		require.Len(t, actualEvents, 1)

		expectedEvent.State = JobStateInProgress
		expectedEvent.UpdatedAt = actualEvents[0].UpdatedAt

		require.Equal(t, expectedEvent, actualEvents[0])

		eventIDs := []uint64{actualEvent.ID}
		ackedIDs, err := queue.Acknowledge(ctx, JobStateFailed, eventIDs)
		require.NoError(t, err)
		require.Equal(t, eventIDs, ackedIDs)
	}
}

func TestPostgresReplicationEventQueue_EnqueueMultiple(t *testing.T) {
	t.Parallel()
	db := testdb.New(t)
	ctx := testhelper.Context(t)

	queue := PostgresReplicationEventQueue{db.DB}

	eventType1 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect-0",
			Params:            nil,
		},
	}

	eventType2 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-2",
			SourceNodeStorage: "",
			VirtualStorage:    "praefect-0",
			Params:            nil,
		},
	}

	eventType3 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-2",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect-1",
			Params:            nil,
		},
	}

	event1, err := queue.Enqueue(ctx, eventType1) // initial event
	require.NoError(t, err)

	expLock1 := LockRow{ID: "praefect-0|gitaly-1|/project/path-1", Acquired: false}
	expLock2 := LockRow{ID: "praefect-0|gitaly-2|/project/path-1", Acquired: false}
	expLock3 := LockRow{ID: "praefect-1|gitaly-1|/project/path-2", Acquired: false}

	expEvent1 := ReplicationEvent{
		ID:      event1.ID,
		State:   "ready",
		Attempt: 3,
		LockID:  "praefect-0|gitaly-1|/project/path-1",
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect-0",
			Params:            nil,
		},
	}

	requireEvents(t, ctx, db, []ReplicationEvent{expEvent1})
	requireLocks(t, ctx, db, []LockRow{expLock1}) // expected a new lock for new event
	db.RequireRowsInTable(t, "replication_queue_job_lock", 0)

	_, err = queue.Enqueue(ctx, eventType1) // repeat of the same event
	require.ErrorAs(t, err, &ReplicationEventExistsError{})

	// doesn't insert the same event again
	requireEvents(t, ctx, db, []ReplicationEvent{expEvent1})
	// expected still one the same lock for repeated event
	requireLocks(t, ctx, db, []LockRow{expLock1})

	event2, err := queue.Enqueue(ctx, eventType2) // event for another target
	require.NoError(t, err)

	expEvent2 := ReplicationEvent{
		ID:      event2.ID,
		State:   JobStateReady,
		Attempt: 3,
		LockID:  "praefect-0|gitaly-2|/project/path-1",
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-2",
			SourceNodeStorage: "",
			VirtualStorage:    "praefect-0",
			Params:            nil,
		},
	}

	requireEvents(t, ctx, db, []ReplicationEvent{expEvent1, expEvent2})
	requireLocks(t, ctx, db, []LockRow{expLock1, expLock2}) // the new lock for another target repeated event

	event3, err := queue.Enqueue(ctx, eventType3) // event for another repo
	require.NoError(t, err)

	expEvent3 := ReplicationEvent{
		ID:      event3.ID,
		State:   JobStateReady,
		Attempt: 3,
		LockID:  "praefect-1|gitaly-1|/project/path-2",
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-2",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect-1",
			Params:            nil,
		},
	}

	requireEvents(t, ctx, db, []ReplicationEvent{expEvent1, expEvent2, expEvent3})
	requireLocks(t, ctx, db, []LockRow{expLock1, expLock2, expLock3}) // the new lock for same target but for another repo

	db.RequireRowsInTable(t, "replication_queue_job_lock", 0) // there is no fetches it must be empty
}

func TestPostgresReplicationEventQueue_Dequeue(t *testing.T) {
	t.Parallel()
	db := testdb.New(t)
	ctx := testhelper.Context(t)

	queue := PostgresReplicationEventQueue{db.DB}

	const virtualStorage = "praefect"
	const targetNodeStorage = "gitaly-1"

	// Start with a list of 5 events
	events := []ReplicationEvent{
		{
			Job: ReplicationJob{
				Change:            UpdateRepo,
				RelativePath:      "/project/path-1",
				TargetNodeStorage: targetNodeStorage,
				SourceNodeStorage: "gitaly-0",
				VirtualStorage:    virtualStorage,
			},
		},
		{
			Job: ReplicationJob{
				Change:            UpdateRepo,
				RelativePath:      "/project/path-2",
				TargetNodeStorage: targetNodeStorage,
				SourceNodeStorage: "gitaly-0",
				VirtualStorage:    virtualStorage,
			},
		},
		{
			Job: ReplicationJob{
				Change:            UpdateRepo,
				RelativePath:      "/project/path-3",
				TargetNodeStorage: targetNodeStorage,
				SourceNodeStorage: "gitaly-0",
				VirtualStorage:    virtualStorage,
			},
		},
		{
			Job: ReplicationJob{
				Change:            UpdateRepo,
				RelativePath:      "/project/path-4",
				TargetNodeStorage: targetNodeStorage,
				SourceNodeStorage: "gitaly-0",
				VirtualStorage:    virtualStorage,
			},
		},
		{
			Job: ReplicationJob{
				Change:            UpdateRepo,
				RelativePath:      "/project/path-5",
				TargetNodeStorage: targetNodeStorage,
				SourceNodeStorage: "gitaly-0",
				VirtualStorage:    virtualStorage,
			},
		},
	}

	// Enqueue each event and make sure no error is returned for each
	for _, e := range events {
		_, err := queue.Enqueue(ctx, e)
		require.NoError(t, err, "failed to fill in event queue")
	}

	var expectedLocks []LockRow
	var expectedJobLocks []JobLockRow

	addToExpectedLocks := func(events []ReplicationEvent) {
		for _, e := range events {
			// As we deque events we acquire lock for processing
			// Validate this event is locked in the DB
			expectedLocks = append(expectedLocks, LockRow{ID: e.LockID, Acquired: true})

			// Validate that the tracked job is also in the DB
			expectedJobLocks = append(expectedJobLocks, JobLockRow{JobID: e.ID, LockID: e.LockID})
		}
	}

	// First, lets enqueue events for a non-existing storage
	// This should return 0 events
	emptyEvents, err := queue.Dequeue(ctx, "praefect", "not existing storage", 5)
	require.NoError(t, err)
	require.Len(t, emptyEvents, 0, "there must be no events dequeued for not existing storage")

	// Dequeue actual events but only limit results to 3. This should leave 2 queued jobs in the queue
	// because we have enqueued 5 events. We will fetch them later
	dequeuedEvents, err := queue.Dequeue(ctx, virtualStorage, targetNodeStorage, 3)
	require.NoError(t, err)
	require.Len(t, dequeuedEvents, 3)

	// Dequeue remaining events. This should return the 2 remaining events in the queue.
	// Let's add a limit of 5 to test that it can return the remaining items event if it less than
	// the set limit.
	dequeuedEventsSecondRound, err := queue.Dequeue(ctx, virtualStorage, targetNodeStorage, 5)
	require.NoError(t, err)
	require.Len(t, dequeuedEventsSecondRound, 2)

	// Dequeue one last time. This should return 0 events
	dequeuedEventsSecondThird, err := queue.Dequeue(ctx, virtualStorage, targetNodeStorage, 3)
	require.NoError(t, err)
	require.Len(t, dequeuedEventsSecondThird, 0)

	// This is a list of all dequeued events from the queue
	receivedEvents := append(dequeuedEvents, dequeuedEventsSecondRound...)

	// Validate events are correctly locked in the DB
	addToExpectedLocks(receivedEvents)
	requireLocks(t, ctx, db, expectedLocks)
	requireJobLocks(t, ctx, db, expectedJobLocks)

	var receivedRelativePath []string
	for _, event := range receivedEvents {
		// lets save the relative path for later validation
		receivedRelativePath = append(receivedRelativePath, event.Job.RelativePath)

		// Every job starts with 3 attempts, so now that we have fetched
		// them, it should be returned with the count set to 2
		require.Equal(t, event.Attempt, 2)

		// Make sure the event is returned with the expected state
		require.Equal(t, event.State, JobState("in_progress"))
	}

	// Validate that we indeed have received each event once
	require.Equal(t, []string{
		"/project/path-1",
		"/project/path-2",
		"/project/path-3",
		"/project/path-4",
		"/project/path-5",
	}, receivedRelativePath)
}

// expected results are listed as literals on purpose to be more explicit about what is going on with data
func TestPostgresReplicationEventQueue_DequeueMultiple(t *testing.T) {
	t.Parallel()
	db := testdb.New(t)
	ctx := testhelper.Context(t)

	queue := PostgresReplicationEventQueue{db.DB}

	eventType1 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	eventType2 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            DeleteRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	eventType3 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-2",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	eventType4 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "backup",
		},
	}

	// events to fill in the queue
	events := []ReplicationEvent{eventType1, eventType2, eventType3, eventType4}
	for i := range events {
		var err error
		events[i], err = queue.Enqueue(ctx, events[i])
		require.NoError(t, err, "failed to fill in event queue")
	}

	// first request to deque
	expectedEvents1 := []ReplicationEvent{events[0], events[1], events[2]}
	expectedJobLocks1 := []JobLockRow{
		{JobID: events[0].ID, LockID: "praefect|gitaly-1|/project/path-1"},
		{JobID: events[1].ID, LockID: "praefect|gitaly-1|/project/path-1"},
		{JobID: events[2].ID, LockID: "praefect|gitaly-1|/project/path-2"},
	}

	// we expect only first two types of events by limiting count to 3
	dequeuedEvents1, err := queue.Dequeue(ctx, "praefect", "gitaly-1", 3)
	require.NoError(t, err)
	require.Len(t, dequeuedEvents1, len(expectedEvents1))
	for i := range dequeuedEvents1 {
		dequeuedEvents1[i].UpdatedAt = nil // it is not possible to determine update_at value as it is generated on UPDATE in database
		expectedEvents1[i].State = JobStateInProgress
		expectedEvents1[i].Attempt--
	}
	require.Equal(t, expectedEvents1, dequeuedEvents1)

	requireLocks(t, ctx, db, []LockRow{
		// there is only one single lock for all fetched events because of their 'repo' and 'target' combination
		{ID: "praefect|gitaly-1|/project/path-1", Acquired: true},
		{ID: "praefect|gitaly-1|/project/path-2", Acquired: true},
		{ID: "backup|gitaly-1|/project/path-1", Acquired: false},
	})
	requireJobLocks(t, ctx, db, expectedJobLocks1)

	// second request to deque
	// there must be only last event fetched from the queue
	expectedEvents2 := []ReplicationEvent{events[3]}
	expectedEvents2[0].State = JobStateInProgress
	expectedEvents2[0].Attempt = 2

	expectedJobLocks2 := []JobLockRow{{JobID: 4, LockID: "backup|gitaly-1|/project/path-1"}}

	dequeuedEvents2, err := queue.Dequeue(ctx, "backup", "gitaly-1", 100500)
	require.NoError(t, err)
	require.Len(t, dequeuedEvents2, 1, "only one event must be fetched from the queue")

	dequeuedEvents2[0].UpdatedAt = nil // it is not possible to determine update_at value as it is generated on UPDATE in database
	require.Equal(t, expectedEvents2, dequeuedEvents2)

	requireLocks(t, ctx, db, []LockRow{
		{ID: "praefect|gitaly-1|/project/path-1", Acquired: true},
		{ID: "praefect|gitaly-1|/project/path-2", Acquired: true},
		{ID: "backup|gitaly-1|/project/path-1", Acquired: true},
	})
	requireJobLocks(t, ctx, db, append(expectedJobLocks1, expectedJobLocks2...))
}

func TestPostgresReplicationEventQueue_DequeueSameStorageOtherRepository(t *testing.T) {
	t.Parallel()
	db := testdb.New(t)
	ctx := testhelper.Context(t)

	queue := PostgresReplicationEventQueue{db.DB}

	eventType1 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	eventType2 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-2",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	_, err := queue.Enqueue(ctx, eventType1)
	require.NoError(t, err, "failed to fill in event queue")

	dequeuedEvents1, err := queue.Dequeue(ctx, "praefect", "gitaly-1", 1)
	require.NoError(t, err)
	require.Len(t, dequeuedEvents1, 1)
	requireLocks(t, ctx, db, []LockRow{
		// there is only one single lock for all fetched events because of their 'repo' and 'target' combination
		{ID: "praefect|gitaly-1|/project/path-1", Acquired: true},
	})
	requireJobLocks(t, ctx, db, []JobLockRow{{JobID: 1, LockID: "praefect|gitaly-1|/project/path-1"}})

	_, err = queue.Enqueue(ctx, eventType2)
	require.NoError(t, err, "failed to fill in event queue")

	dequeuedEvents2, err := queue.Dequeue(ctx, "praefect", "gitaly-1", 1)
	require.NoError(t, err)
	require.Len(t, dequeuedEvents2, 1)
	requireLocks(t, ctx, db, []LockRow{
		{ID: "praefect|gitaly-1|/project/path-1", Acquired: true},
		{ID: "praefect|gitaly-1|/project/path-2", Acquired: true},
	})
	requireJobLocks(t, ctx, db, []JobLockRow{
		{JobID: 1, LockID: "praefect|gitaly-1|/project/path-1"},
		{JobID: 2, LockID: "praefect|gitaly-1|/project/path-2"},
	})
}

func TestPostgresReplicationEventQueue_Acknowledge(t *testing.T) {
	t.Parallel()
	db := testdb.New(t)
	ctx := testhelper.Context(t)

	queue := PostgresReplicationEventQueue{db.DB}

	event := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	event, err := queue.Enqueue(ctx, event)
	require.NoError(t, err, "failed to fill in event queue")

	actual, err := queue.Dequeue(ctx, event.Job.VirtualStorage, event.Job.TargetNodeStorage, 100)
	require.NoError(t, err)

	// as we deque events we acquire lock for processing
	requireLocks(t, ctx, db, []LockRow{{ID: event.LockID, Acquired: true}})
	requireJobLocks(t, ctx, db, []JobLockRow{{JobID: event.ID, LockID: event.LockID}})

	acknowledged, err := queue.Acknowledge(ctx, JobStateCompleted, []uint64{actual[0].ID, 100500})
	require.NoError(t, err)
	require.Equal(t, []uint64{actual[0].ID}, acknowledged)

	event.State = JobStateCompleted
	event.Attempt = 2
	// events acknowledged with 'completed' or 'dead' states expected to be removed
	db.RequireRowsInTable(t, "replication_queue", 0)
	// all associated with acknowledged event tracking bindings between lock and event must be removed
	db.RequireRowsInTable(t, "replication_queue_job_lock", 0)
	// lock must be released as the event was acknowledged and there are no other events left protected under this lock
	requireLocks(t, ctx, db, []LockRow{{ID: event.LockID, Acquired: false}})
}

func TestPostgresReplicationEventQueue_AcknowledgeMultiple(t *testing.T) {
	t.Parallel()
	db := testdb.New(t)
	ctx := testhelper.Context(t)

	queue := PostgresReplicationEventQueue{db.DB}

	eventType1 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	eventType2 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            DeleteRepo,
			RelativePath:      "/project/path-2",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	eventType3 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-3",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	eventType4 := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-2",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	events := []ReplicationEvent{eventType1, eventType2, eventType3, eventType4}
	for i := range events {
		var err error
		events[i], err = queue.Enqueue(ctx, events[i])
		require.NoError(t, err, "failed to fill in event queue")
	}

	// we expect only first two types events by limiting count to 2
	dequeuedEvents1, err := queue.Dequeue(ctx, "praefect", "gitaly-1", 2)
	require.NoError(t, err)
	require.Len(t, dequeuedEvents1, 2)
	requireLocks(t, ctx, db, []LockRow{
		{ID: events[0].LockID, Acquired: true},
		{ID: events[1].LockID, Acquired: true},
		{ID: events[2].LockID, Acquired: false},
		{ID: events[3].LockID, Acquired: false},
	})
	requireJobLocks(t, ctx, db, []JobLockRow{
		{JobID: events[0].ID, LockID: events[0].LockID},
		{JobID: events[1].ID, LockID: events[1].LockID},
	})

	// release lock for events of second type
	acknowledge1, err := queue.Acknowledge(ctx, JobStateFailed, []uint64{events[1].ID})
	require.NoError(t, err)
	require.Equal(t, []uint64{2}, acknowledge1)
	requireLocks(t, ctx, db, []LockRow{
		{ID: events[0].LockID, Acquired: true},
		{ID: events[1].LockID, Acquired: false},
		{ID: events[2].LockID, Acquired: false},
		{ID: events[3].LockID, Acquired: false},
	})
	requireJobLocks(t, ctx, db, []JobLockRow{
		{JobID: events[0].ID, LockID: events[0].LockID},
	})

	dequeuedEvents2, err := queue.Dequeue(ctx, "praefect", "gitaly-1", 3)
	require.NoError(t, err)
	require.Len(t, dequeuedEvents2, 2, "expected: events of type 2 ('failed' will  be fetched for retry)")
	requireLocks(t, ctx, db, []LockRow{
		{ID: events[0].LockID, Acquired: true},
		{ID: events[1].LockID, Acquired: true},
		{ID: events[2].LockID, Acquired: true},
		{ID: events[3].LockID, Acquired: false},
	})
	requireJobLocks(t, ctx, db, []JobLockRow{
		{JobID: events[0].ID, LockID: events[0].LockID},
		{JobID: events[1].ID, LockID: events[1].LockID},
		{JobID: events[2].ID, LockID: events[2].LockID},
	})

	// creation of the new event that is equal to those already dequeue and processed
	// it is used to verify that the event created after consuming events from queue won't be marked
	// with previously created events as it may cause delay in replication
	newEvent, err := queue.Enqueue(ctx, eventType1)
	require.NoError(t, err)

	acknowledge2, err := queue.Acknowledge(ctx, JobStateCompleted, []uint64{events[0].ID, events[2].ID})
	require.NoError(t, err)
	require.Equal(t, []uint64{events[0].ID, events[2].ID}, acknowledge2)
	requireLocks(t, ctx, db, []LockRow{
		{ID: events[0].LockID, Acquired: false},
		{ID: events[1].LockID, Acquired: true},
		{ID: events[2].LockID, Acquired: false},
		{ID: events[3].LockID, Acquired: false},
	})
	requireJobLocks(t, ctx, db, []JobLockRow{
		{JobID: events[1].ID, LockID: events[1].LockID},
	})
	db.RequireRowsInTable(t, "replication_queue", 3)

	dequeuedEvents3, err := queue.Dequeue(ctx, "praefect", "gitaly-2", 3)
	require.NoError(t, err)
	require.Len(t, dequeuedEvents3, 1, "expected: event of type 4")
	requireLocks(t, ctx, db, []LockRow{
		{ID: events[0].LockID, Acquired: false},
		{ID: events[1].LockID, Acquired: true},
		{ID: events[2].LockID, Acquired: false},
		{ID: events[3].LockID, Acquired: true},
	})
	requireJobLocks(t, ctx, db, []JobLockRow{
		{JobID: events[1].ID, LockID: events[1].LockID},
		{JobID: events[3].ID, LockID: events[3].LockID},
	})

	acknowledged3, err := queue.Acknowledge(ctx, JobStateCompleted, []uint64{events[1].ID, events[3].ID})
	require.NoError(t, err)
	require.Equal(t, []uint64{events[1].ID, events[3].ID}, acknowledged3)
	requireLocks(t, ctx, db, []LockRow{
		{ID: events[0].LockID, Acquired: false},
		{ID: events[1].LockID, Acquired: false},
		{ID: events[2].LockID, Acquired: false},
		{ID: events[3].LockID, Acquired: false},
	})
	requireJobLocks(t, ctx, db, nil)
	db.RequireRowsInTable(t, "replication_queue", 1)

	acknowledge4, err := queue.Acknowledge(ctx, JobStateCompleted, []uint64{newEvent.ID})
	require.NoError(t, err)
	require.Equal(t, ([]uint64)(nil), acknowledge4) // event that was not dequeued can't be acknowledged
	db.RequireRowsInTable(t, "replication_queue", 1)

	var newEventState string
	require.NoError(t, db.QueryRowContext(ctx, "SELECT state FROM replication_queue WHERE id = $1", newEvent.ID).Scan(&newEventState))
	require.Equal(t, "ready", newEventState, "no way to acknowledge event that is not in in_progress state(was not dequeued)")
	requireLocks(t, ctx, db, []LockRow{
		{ID: events[0].LockID, Acquired: false},
		{ID: events[1].LockID, Acquired: false},
		{ID: events[2].LockID, Acquired: false},
		{ID: events[3].LockID, Acquired: false},
	})
	requireJobLocks(t, ctx, db, nil)
}

func TestPostgresReplicationEventQueue_StartHealthUpdate(t *testing.T) {
	t.Parallel()
	eventType1 := ReplicationEvent{Job: ReplicationJob{
		Change:            UpdateRepo,
		VirtualStorage:    "vs-1",
		TargetNodeStorage: "s-1",
		SourceNodeStorage: "s-0",
		RelativePath:      "/path/1",
	}}

	eventType2 := eventType1
	eventType2.Job.RelativePath = "/path/2"

	eventType3 := eventType1
	eventType3.Job.VirtualStorage = "vs-2"

	eventType4 := eventType1
	eventType4.Job.TargetNodeStorage = "s-2"

	db := testdb.New(t)

	t.Run("no events is valid", func(t *testing.T) {
		// 'qc' is not initialized, so the test will fail if there will be an attempt to make SQL operation
		queue := NewPostgresReplicationEventQueue(nil)
		ctx := testhelper.Context(t)

		require.NoError(t, queue.StartHealthUpdate(ctx, nil, nil))
	})

	t.Run("can be terminated by the passed in context", func(t *testing.T) {
		ctx, cancel := context.WithCancel(testhelper.Context(t))

		// 'qc' is not initialized, so the test will fail if there will be an attempt to make SQL operation
		queue := NewPostgresReplicationEventQueue(nil)
		cancel()
		require.NoError(t, queue.StartHealthUpdate(ctx, nil, []ReplicationEvent{eventType1}))
	})

	t.Run("stops after first error", func(t *testing.T) {
		db.TruncateAll(t)
		ctx := testhelper.Context(t)

		qc, err := db.DB.BeginTx(ctx, nil)
		require.NoError(t, err)
		require.NoError(t, qc.Rollback())

		// 'qc' is initialized with invalid connection (transaction is finished), so operations on it will fail
		queue := NewPostgresReplicationEventQueue(qc)

		trigger := make(chan time.Time, 1)
		trigger <- time.Time{}

		require.Error(t, queue.StartHealthUpdate(ctx, trigger, []ReplicationEvent{eventType1}))
	})

	t.Run("stops if nothing to update (extended coverage)", func(t *testing.T) {
		db.TruncateAll(t)
		ctx := testhelper.Context(t)

		done := make(chan struct{})
		queue := NewPostgresReplicationEventQueue(db)
		go func() {
			trigger := make(chan time.Time)
			close(trigger)

			defer close(done)
			assert.NoError(t, queue.StartHealthUpdate(ctx, trigger, []ReplicationEvent{eventType1}))
		}()

		select {
		case <-done:
			return // happy path
		case <-time.After(time.Second):
			require.FailNow(t, "method should return almost immediately as there is nothing to process")
		}
	})

	t.Run("triggers all passed in events", func(t *testing.T) {
		db.TruncateAll(t)

		var wg sync.WaitGroup
		ctx, cancel := context.WithCancel(testhelper.Context(t))
		defer func() {
			cancel()
			wg.Wait()
		}()

		queue := NewPostgresReplicationEventQueue(db)
		events := []ReplicationEvent{eventType1, eventType2, eventType3, eventType4}
		for i := range events {
			var err error
			events[i], err = queue.Enqueue(ctx, events[i])
			require.NoError(t, err, "failed to fill in event queue")
		}

		dequeuedEventsToTrigger, err := queue.Dequeue(ctx, eventType1.Job.VirtualStorage, eventType1.Job.TargetNodeStorage, 10)
		require.NoError(t, err)
		require.Len(t, dequeuedEventsToTrigger, 2, "eventType3 and eventType4 should not be fetched")
		ids := []uint64{dequeuedEventsToTrigger[0].ID, dequeuedEventsToTrigger[1].ID}

		dequeuedEventsUntriggered, err := queue.Dequeue(ctx, eventType3.Job.VirtualStorage, eventType3.Job.TargetNodeStorage, 10)
		require.NoError(t, err)
		require.Len(t, dequeuedEventsUntriggered, 1, "only eventType3 should be fetched")

		initialJobLocks := fetchJobLocks(t, ctx, db)

		trigger := make(chan time.Time, 1)
		wg.Add(1)
		go func() {
			defer wg.Done()
			assert.NoError(t, queue.StartHealthUpdate(ctx, trigger, dequeuedEventsToTrigger))
		}()

		trigger <- time.Time{}
		// once this consumed we are sure that the previous update has been executed
		trigger <- time.Time{}

		updatedJobLocks := fetchJobLocks(t, ctx, db)
		for i := range initialJobLocks {
			if updatedJobLocks[i].JobID == dequeuedEventsUntriggered[0].ID {
				require.Equal(t, initialJobLocks[i].TriggeredAt, updatedJobLocks[i].TriggeredAt, "no update expected as it was not submitted")
			} else {
				require.GreaterOrEqual(t, updatedJobLocks[i].TriggeredAt.UnixNano(), initialJobLocks[i].TriggeredAt.UnixNano())
			}
		}

		ackIDs, err := queue.Acknowledge(ctx, JobStateFailed, ids)
		require.NoError(t, err)
		require.ElementsMatch(t, ackIDs, ids)

		require.Len(t, fetchJobLocks(t, ctx, db), 1, "bindings should be removed after acknowledgment")
	})
}

func TestPostgresReplicationEventQueue_AcknowledgeStale(t *testing.T) {
	t.Parallel()
	ctx := testhelper.Context(t)

	eventType := ReplicationEvent{Job: ReplicationJob{
		Change:            UpdateRepo,
		RelativePath:      "/project/path-1",
		TargetNodeStorage: "gitaly-1",
		SourceNodeStorage: "gitaly-0",
		VirtualStorage:    "praefect-1",
	}}

	eventType1 := eventType

	eventType2 := eventType
	eventType2.Job.VirtualStorage = "praefect-2"

	eventType3 := eventType2
	eventType3.Job.RelativePath = "/project/path-2"
	eventType3.Job.TargetNodeStorage = "gitaly-2"

	eventType4 := eventType3
	eventType4.Job.TargetNodeStorage = "gitaly-3"

	db := testdb.New(t)

	t.Run("no stale jobs yet", func(t *testing.T) {
		db.TruncateAll(t)
		source := NewPostgresReplicationEventQueue(db)

		event, err := source.Enqueue(ctx, eventType1)
		require.NoError(t, err)

		devents, err := source.Dequeue(ctx, event.Job.VirtualStorage, event.Job.TargetNodeStorage, 1)
		require.NoError(t, err)

		// events triggered just now (< 1 sec ago), so nothing considered stale
		n, err := source.AcknowledgeStale(ctx, time.Second)
		require.NoError(t, err)
		requireEvents(t, ctx, db, devents)
		require.Equal(t, n, int64(0))
	})

	t.Run("jobs considered stale only at 'in_progress' state", func(t *testing.T) {
		db.TruncateAll(t)
		source := NewPostgresReplicationEventQueue(db)

		// move event to 'ready' state
		event1, err := source.Enqueue(ctx, eventType1)
		require.NoError(t, err)

		// move event to 'failed' state
		event2, err := source.Enqueue(ctx, eventType2)
		require.NoError(t, err)
		devents2, err := source.Dequeue(ctx, event2.Job.VirtualStorage, event2.Job.TargetNodeStorage, 1)
		require.NoError(t, err)
		require.Equal(t, event2.ID, devents2[0].ID)
		_, err = source.Acknowledge(ctx, JobStateFailed, []uint64{devents2[0].ID})
		require.NoError(t, err)

		// move event to 'dead' state
		event3, err := source.Enqueue(ctx, eventType3)
		require.NoError(t, err)
		devents3, err := source.Dequeue(ctx, event3.Job.VirtualStorage, event3.Job.TargetNodeStorage, 1)
		require.NoError(t, err)
		require.Equal(t, event3.ID, devents3[0].ID)
		_, err = source.Acknowledge(ctx, JobStateDead, []uint64{devents3[0].ID})
		require.NoError(t, err)

		event4, err := source.Enqueue(ctx, eventType4)
		require.NoError(t, err)
		devents4, err := source.Dequeue(ctx, event4.Job.VirtualStorage, event4.Job.TargetNodeStorage, 1)
		require.NoError(t, err)

		n, err := source.AcknowledgeStale(ctx, time.Microsecond)
		require.NoError(t, err)

		devents2[0].State = JobStateFailed
		devents4[0].Attempt = 2
		devents4[0].State = JobStateFailed
		requireEvents(t, ctx, db, []ReplicationEvent{event1, devents2[0], devents4[0]})
		require.Equal(t, n, int64(1))
	})

	t.Run("stale jobs updated for all virtual storages and storages at once", func(t *testing.T) {
		db.TruncateAll(t)
		source := NewPostgresReplicationEventQueue(db)

		var events []ReplicationEvent
		for _, eventType := range []ReplicationEvent{eventType1, eventType2, eventType3} {
			event, err := source.Enqueue(ctx, eventType)
			require.NoError(t, err)
			devents, err := source.Dequeue(ctx, event.Job.VirtualStorage, event.Job.TargetNodeStorage, 1)
			require.NoError(t, err)
			events = append(events, devents...)
		}

		for event, i := events[0], 0; i < 2; i++ { // consume all processing attempts to verify that state will be changed to 'dead'
			_, err := source.Acknowledge(ctx, JobStateFailed, []uint64{event.ID})
			require.NoError(t, err)
			_, err = source.Dequeue(ctx, event.Job.VirtualStorage, event.Job.TargetNodeStorage, 1)
			require.NoError(t, err)
		}

		n, err := source.AcknowledgeStale(ctx, time.Microsecond)
		require.NoError(t, err)
		require.Equal(t, n, int64(3))

		var exp []ReplicationEvent
		// The first event should be removed from table as its state changed to 'dead'.
		for _, e := range events[1:] {
			e.State = JobStateFailed
			exp = append(exp, e)
		}

		requireEvents(t, ctx, db, exp)
	})
}

// Check if the queue returns the expected error when adding a duplicate
// replication event.
func TestPostgresReplicationEventQueue_DuplicateEvent(t *testing.T) {
	t.Parallel()

	initialEvent := ReplicationEvent{
		Job: ReplicationJob{
			Change:            UpdateRepo,
			RelativePath:      "/project/path-1",
			TargetNodeStorage: "gitaly-1",
			SourceNodeStorage: "gitaly-0",
			VirtualStorage:    "praefect",
			Params:            nil,
		},
	}

	for _, tc := range []struct {
		desc          string
		event         ReplicationEvent
		expectedErr   error
		expectedEvent *ReplicationEvent
		expectedLock  *LockRow
	}{
		{
			desc:  "throws error when same event added again",
			event: initialEvent,
			expectedErr: ReplicationEventExistsError{
				state:             "",
				virtualStorage:    initialEvent.Job.VirtualStorage,
				targetNodeStorage: initialEvent.Job.TargetNodeStorage,
				relativePath:      initialEvent.Job.RelativePath,
			},
		},
		{
			desc: "Adds new event to the queue",
			event: ReplicationEvent{
				LockID: "praefect|gitaly-2|/project/path-2",
				Job: ReplicationJob{
					Change:            UpdateRepo,
					RelativePath:      "/project/path-2",
					TargetNodeStorage: "gitaly-2",
					SourceNodeStorage: "gitaly-1",
					VirtualStorage:    "praefect",
					Params:            nil,
				},
			},
			expectedEvent: &ReplicationEvent{
				ID:      2,
				State:   JobStateReady,
				Attempt: 3,
				LockID:  "praefect|gitaly-2|/project/path-2",
				Job: ReplicationJob{
					Change:            UpdateRepo,
					RelativePath:      "/project/path-2",
					TargetNodeStorage: "gitaly-2",
					SourceNodeStorage: "gitaly-1",
					VirtualStorage:    "praefect",
					Params:            nil,
				},
			},
			expectedLock: &LockRow{ID: "praefect|gitaly-2|/project/path-2", Acquired: false},
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			t.Parallel()

			db := testdb.New(t)
			ctx := testhelper.Context(t)

			queue := PostgresReplicationEventQueue{db.DB}

			actualEvent, err := queue.Enqueue(ctx, initialEvent)
			require.NoError(t, err)
			// we need to setup it to default because it is not possible to get it beforehand for expected
			actualEvent.CreatedAt = time.Time{}

			initialExpectedLock := LockRow{ID: "praefect|gitaly-1|/project/path-1", Acquired: false}
			initialExpectedEvent := ReplicationEvent{
				ID:      1,
				State:   JobStateReady,
				Attempt: 3,
				LockID:  "praefect|gitaly-1|/project/path-1",
				Job: ReplicationJob{
					Change:            UpdateRepo,
					RelativePath:      "/project/path-1",
					TargetNodeStorage: "gitaly-1",
					SourceNodeStorage: "gitaly-0",
					VirtualStorage:    "praefect",
					Params:            nil,
				},
			}

			require.Equal(t, initialExpectedEvent, actualEvent)
			requireEvents(t, ctx, db, []ReplicationEvent{initialExpectedEvent})
			// expected a new lock for new event
			requireLocks(t, ctx, db, []LockRow{initialExpectedLock})
			db.RequireRowsInTable(t, "replication_queue_job_lock", 0)

			_, err = queue.Enqueue(ctx, tc.event)
			require.Equal(t, tc.expectedErr, err)

			if tc.expectedEvent != nil {
				requireEvents(t, ctx, db, []ReplicationEvent{initialExpectedEvent, *tc.expectedEvent})
				requireLocks(t, ctx, db, []LockRow{initialExpectedLock, *tc.expectedLock})
			}
		})
	}
}

func requireEvents(t *testing.T, ctx context.Context, db testdb.DB, expected []ReplicationEvent) {
	t.Helper()

	// as it is not possible to expect exact time of entity creation/update we do not fetch it from database
	// and we do not take it into account from expected values.
	exp := make([]ReplicationEvent, len(expected)) // make a copy to avoid side effects for passed values
	for i, e := range expected {
		exp[i] = e
		// set to default values as they would not be initialized from database
		exp[i].CreatedAt = time.Time{}
		exp[i].UpdatedAt = nil
	}

	sqlStmt := `SELECT id, state, attempt, lock_id, job FROM replication_queue ORDER BY id`
	rows, err := db.QueryContext(ctx, sqlStmt)
	require.NoError(t, err)

	actual, err := scanReplicationEvents(rows)
	require.NoError(t, err)
	require.Equal(t, exp, actual)
}

// LockRow exists only for testing purposes and represents entries from replication_queue_lock table.
type LockRow struct {
	ID       string
	Acquired bool
}

func requireLocks(t *testing.T, ctx context.Context, db testdb.DB, expected []LockRow) {
	t.Helper()

	sqlStmt := `SELECT id, acquired FROM replication_queue_lock`
	rows, err := db.QueryContext(ctx, sqlStmt)
	require.NoError(t, err)
	defer func() { require.NoError(t, rows.Close(), "completion of result fetching") }()

	var actual []LockRow
	for rows.Next() {
		var entry LockRow
		require.NoError(t, rows.Scan(&entry.ID, &entry.Acquired), "failed to scan entry")
		actual = append(actual, entry)
	}
	require.NoError(t, rows.Err(), "completion of result loop scan")
	require.ElementsMatch(t, expected, actual)
}

// JobLockRow exists only for testing purposes and represents entries from replication_queue_job_lock table.
type JobLockRow struct {
	JobID       uint64
	LockID      string
	TriggeredAt time.Time
}

func requireJobLocks(t *testing.T, ctx context.Context, db testdb.DB, expected []JobLockRow) {
	t.Helper()

	actual := fetchJobLocks(t, ctx, db)
	for i := range actual {
		actual[i].TriggeredAt = time.Time{}
	}
	require.ElementsMatch(t, expected, actual)
}

func fetchJobLocks(t *testing.T, ctx context.Context, db testdb.DB) []JobLockRow {
	t.Helper()
	sqlStmt := `SELECT job_id, lock_id, triggered_at FROM replication_queue_job_lock ORDER BY job_id`
	rows, err := db.QueryContext(ctx, sqlStmt)
	require.NoError(t, err)
	defer func() { require.NoError(t, rows.Close(), "completion of result fetching") }()

	var entries []JobLockRow
	for rows.Next() {
		var entry JobLockRow
		require.NoError(t, rows.Scan(&entry.JobID, &entry.LockID, &entry.TriggeredAt), "failed to scan entry")
		entries = append(entries, entry)
	}
	require.NoError(t, rows.Err(), "completion of result loop scan")

	return entries
}