Skip to content

Git designed for AI agents.

12.7 KB
Raw Blame
package testdb

import (
	"context"
	"database/sql"
	"errors"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/feder-cr/git-ai/internal/praefect/config"
	"github.com/feder-cr/git-ai/internal/praefect/datastore/glsql"
	"github.com/feder-cr/git-ai/internal/praefect/datastore/migrations"
	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/google/uuid"
	migrate "github.com/rubenv/sql-migrate"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

const (
	// testDBMaxOpenConnections caps the number of open connections each test
	// database pool may hold. The tests create many databases concurrently, so
	// keeping the per-database value low will prevent us from exhausting the
	// Postgres server.
	testDBMaxOpenConnections = 10
	// testDBMaxIdleConnections caps the number of idle connections retained by
	// each test database. Limiting this will significantly reduce the number
	// of connections held across concurrently-running tests.
	testDBMaxIdleConnections = 1
)

// TxWrapper is a simple wrapper around *sql.Tx.
type TxWrapper struct {
	*sql.Tx
}

// Rollback executes Rollback operation on the wrapped *sql.Tx if it is set.
// After execution is sets Tx to nil to prevent errors on the repeated invocations (useful
// for testing when Rollback is deferred).
func (txw *TxWrapper) Rollback(tb testing.TB) {
	tb.Helper()
	if txw.Tx != nil {
		require.NoError(tb, txw.Tx.Rollback())
		txw.Tx = nil
	}
}

// Commit executes Commit operation on the wrapped *sql.Tx if it is set.
// After execution is sets Tx to nil to prevent errors on the deferred invocations (useful
// for testing when Rollback is deferred).
func (txw *TxWrapper) Commit(tb testing.TB) {
	tb.Helper()
	if txw.Tx != nil {
		require.NoError(tb, txw.Tx.Commit())
		txw.Tx = nil
	}
}

// DB is a helper struct that should be used only for testing purposes.
type DB struct {
	*sql.DB
	// Name is a name of the database.
	Name string
}

// Begin starts a new transaction and returns it wrapped into TxWrapper.
func (db DB) Begin(tb testing.TB) *TxWrapper {
	tb.Helper()
	ctx := testhelper.Context(tb)
	tx, err := db.DB.BeginTx(ctx, nil)
	require.NoError(tb, err)
	return &TxWrapper{Tx: tx}
}

// Truncate removes all data from the list of tables and restarts identities for them.
func (db DB) Truncate(tb testing.TB, tables ...string) {
	tb.Helper()
	ctx := testhelper.Context(tb)

	for _, table := range tables {
		_, err := db.DB.ExecContext(ctx, "DELETE FROM "+table)
		require.NoError(tb, err, "database cleanup failed: %s", tables)
	}

	_, err := db.DB.ExecContext(ctx, "SELECT setval(relname::TEXT, 1, false) from pg_class where relkind = 'S'")
	require.NoError(tb, err, "database cleanup failed: %s", tables)
}

// RequireRowsInTable verifies that `tname` table has `n` amount of rows in it.
func (db DB) RequireRowsInTable(tb testing.TB, tname string, n int) {
	tb.Helper()
	ctx := testhelper.Context(tb)

	var count int
	require.NoError(tb, db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tname).Scan(&count))
	require.Equal(tb, n, count, "unexpected amount of rows in table: %d instead of %d", count, n)
}

// TruncateAll removes all data from known set of tables.
func (db DB) TruncateAll(tb testing.TB) {
	db.Truncate(tb,
		"replication_queue_job_lock",
		"replication_queue",
		"replication_queue_lock",
		"node_status",
		"shard_primaries",
		"storage_repositories",
		"repositories",
		"virtual_storages",
		"repository_assignments",
		"storage_cleanups",
		"pending_transactions",
	)
}

// MustExec executes `q` with `args` and verifies there are no errors.
func (db DB) MustExec(tb testing.TB, q string, args ...interface{}) {
	ctx := testhelper.Context(tb)
	_, err := db.DB.ExecContext(ctx, q, args...)
	require.NoError(tb, err)
}

// Close removes schema if it was used and releases connection pool.
func (db DB) Close() error {
	if err := db.DB.Close(); err != nil {
		return errors.New("failed to release connection pool: " + err.Error())
	}
	return nil
}

// New returns a wrapper around the database connection pool.
// Must be used only for testing.
// The new database with empty relations will be created for each call of this function.
// It uses env vars:
//
//	PGHOST - required, URL/socket/dir
//	PGPORT - required, binding port
//	PGUSER - optional, user - `$ whoami` would be used if not provided
//
// Once the test is completed the database will be dropped on test cleanup execution.
func New(tb testing.TB) DB {
	tb.Helper()
	database := "praefect_" + strings.ReplaceAll(uuid.New().String(), "-", "")
	return DB{DB: initPraefectDB(tb, database), Name: database}
}

// GetConfig returns the database configuration determined by
// environment variables.  See NewDB() for the list of variables.
func GetConfig(tb testing.TB, database string) config.DB {
	env := getDatabaseEnvironment(tb)

	require.Contains(tb, env, "PGHOST", "PGHOST env var expected to be provided to connect to Postgres database")
	require.Contains(tb, env, "PGPORT", "PGHOST env var expected to be provided to connect to Postgres database")

	portNumber, err := strconv.Atoi(env["PGPORT"])
	require.NoError(tb, err, "PGPORT must be a port number of the Postgres database listens for incoming connections")

	// connect to 'postgres' database first to re-create testing database from scratch
	conf := config.DB{
		Host:    env["PGHOST"],
		Port:    portNumber,
		DBName:  database,
		SSLMode: "disable",
		User:    env["PGUSER"],
		SessionPooled: config.DBConnection{
			Host: env["PGHOST"],
			Port: portNumber,
		},
	}

	if bouncerHost, ok := env["PGHOST_PGBOUNCER"]; ok {
		conf.Host = bouncerHost
	}

	if bouncerPort, ok := env["PGPORT_PGBOUNCER"]; ok {
		bouncerPortNumber, err := strconv.Atoi(bouncerPort)
		require.NoError(tb, err, "PGPORT_PGBOUNCER must be a port number of the PgBouncer")
		conf.Port = bouncerPortNumber
	}

	return conf
}

func requireSQLOpen(tb testing.TB, dbCfg config.DB, direct bool) *sql.DB {
	tb.Helper()
	ctx := testhelper.Context(tb)
	db, err := sql.Open("pgx", glsql.DSN(dbCfg, direct))
	require.NoErrorf(tb, err, "failed to connect to %q database", dbCfg.DBName)
	// Limit connections per database pool. The tests are highly concurrent and can exhaust the
	// Postgres connection limit.
	db.SetMaxOpenConns(testDBMaxOpenConnections)
	db.SetMaxIdleConns(testDBMaxIdleConnections)
	if !assert.NoErrorf(tb, db.PingContext(ctx), "failed to communicate with %q database", dbCfg.DBName) {
		require.NoErrorf(tb, db.Close(), "release connection to the %q database", dbCfg.DBName)
	}
	return db
}

func requireTerminateAllConnections(tb testing.TB, db *sql.DB, database string) {
	tb.Helper()
	ctx := testhelper.Context(tb)
	_, err := db.ExecContext(ctx, "SELECT PG_TERMINATE_BACKEND(pid) FROM PG_STAT_ACTIVITY WHERE datname = $1", database)
	require.NoError(tb, err)

	// Once the pg_terminate_backend has completed, we may need to wait before the connections
	// are fully released. pg_terminate_backend will return true as long as the signal was
	// sent successfully, but the backend needs to respond to the signal to close the connection.
	// TODO: In Postgre 14, pg_terminate_backend takes an optional timeout argument that makes it a blocking
	// call. https://gitlab.com/gitlab-org/gitaly/-/issues/3937 tracks the refactor work to  remove this
	// require.Eventuallyf call in favor of passing in a timeout to pg_terminate_backend
	require.Eventuallyf(tb, func() bool {
		var openConnections int
		require.NoError(tb, db.QueryRowContext(
			ctx,
			`SELECT COUNT(*) FROM pg_stat_activity
				WHERE datname = $1 AND pid != pg_backend_pid()`, database).
			Scan(&openConnections))
		return openConnections == 0
	}, 20*time.Second, 10*time.Millisecond, "wait for all connections to be terminated")
}

func initPraefectDB(tb testing.TB, database string) *sql.DB {
	tb.Helper()
	ctx := testhelper.Context(tb)

	postgresCfg := GetConfig(tb, "postgres")

	postgresDB := requireSQLOpen(tb, postgresCfg, true)
	tb.Cleanup(func() {
		testhelper.MustClose(tb, postgresDB)
	})

	_, err := postgresDB.ExecContext(ctx, "CREATE DATABASE "+database+" WITH ENCODING 'UTF8'")
	require.NoErrorf(tb, err, "failed to create %q database", database)
	tb.Cleanup(func() {
		if _, ok := getDatabaseEnvironment(tb)["PGHOST_PGBOUNCER"]; ok {
			pgbouncerCfg := postgresCfg
			// This database name will connect us to the special admin console.
			pgbouncerCfg.DBName = "pgbouncer"

			// We cannot use `requireSQLOpen()` because it would ping the database,
			// which is not supported by the PgBouncer admin console.
			pgbouncerDB, err := sql.Open("pgx", glsql.DSN(pgbouncerCfg, false))
			require.NoError(tb, err)
			defer testhelper.MustClose(tb, pgbouncerDB)

			// Trying to release connections like we do with the "normal" Postgres
			// database regularly results in flaky tests with PgBouncer given that the
			// connections are seemingly never released. Instead, we kill PgBouncer
			// connections by connecting to its admin console and using the KILL
			// command, which instructs it to kill all client and server connections.
			_, err = pgbouncerDB.ExecContext(ctx, "KILL "+database)
			require.NoError(tb, err, "killing PgBouncer connections")
		}

		// We need to force-terminate open connections as for the tasks that use PgBouncer
		// the actual client connected to the database is a PgBouncer and not a test that is
		// running.
		requireTerminateAllConnections(tb, postgresDB, database)

		_, err := postgresDB.ExecContext(ctx, "DROP DATABASE "+database)
		require.NoErrorf(tb, err, "failed to drop %q database", database)
	})

	praefectDBCfg := GetConfig(tb, database)
	praefectDB := requireSQLOpen(tb, praefectDBCfg, false)
	tb.Cleanup(func() {
		if err := praefectDB.Close(); !errors.Is(err, net.ErrClosed) {
			require.NoErrorf(tb, err, "release connection to the %q database", database)
		}
	})

	_, err = glsql.Migrate(praefectDB, false)
	require.NoErrorf(tb, err, "failed to run database migration on %q", database)

	return praefectDB
}

var (
	// Running `gdk env` takes about 250ms on my system and is thus comparatively slow. When
	// running with Praefect as proxy, this time adds up and may thus slow down tests by quite a
	// margin. We thus amortize these costs by only running it once.
	databaseEnvOnce sync.Once
	databaseEnv     map[string]string
)

func getDatabaseEnvironment(tb testing.TB) map[string]string {
	ctx := testhelper.Context(tb)

	databaseEnvOnce.Do(func() {
		envvars := map[string]string{}

		// We only process output if `gdk env` returned success. If it didn't, we simply assume that
		// we are not running in a GDK environment and will try to extract variables from the
		// environment instead.
		if output, err := exec.CommandContext(ctx, "gdk", "env").Output(); err == nil {
			for _, line := range strings.Split(string(output), "\n") {
				const prefix = "export "
				if !strings.HasPrefix(line, prefix) {
					continue
				}

				split := strings.SplitN(strings.TrimPrefix(line, prefix), "=", 2)
				if len(split) != 2 {
					continue
				}

				envvars[split[0]] = split[1]
			}
		}

		for _, key := range []string{"PGHOST", "PGPORT", "PGUSER", "PGHOST_PGBOUNCER", "PGPORT_PGBOUNCER"} {
			if _, ok := envvars[key]; !ok {
				value, ok := os.LookupEnv(key)
				if ok {
					envvars[key] = value
				}
			}
		}

		databaseEnv = envvars
	})

	return databaseEnv
}

// WaitForBlockedQuery is a helper that waits until a blocked query matching the prefix is present in the
// database. This is useful for ensuring another transaction is blocking a query when testing concurrent
// execution of multiple queries.
func WaitForBlockedQuery(tb testing.TB, ctx context.Context, db glsql.Querier, queryPrefix string) {
	tb.Helper()

	for {
		var queryBlocked bool
		require.NoError(tb, db.QueryRowContext(ctx, `
			SELECT EXISTS (
				SELECT FROM pg_stat_activity
				WHERE TRIM(e'\n' FROM query) LIKE $1
				AND state = 'active'
				AND wait_event_type = 'Lock'
				AND datname = current_database()
			)
		`, queryPrefix+"%").Scan(&queryBlocked))

		if queryBlocked {
			return
		}

		retry := time.NewTimer(time.Millisecond)
		select {
		case <-ctx.Done():
			retry.Stop()
			return
		case <-retry.C:
		}
	}
}

// SetMigrations ensures the requested number of migrations are up and the remainder are down.
func SetMigrations(tb testing.TB, db DB, cfg config.Config, up int) {
	// Ensure all migrations are up first
	_, err := glsql.Migrate(db.DB, true)
	require.NoError(tb, err)

	migrationCt := len(migrations.All())

	if up < migrationCt {
		down := migrationCt - up

		migrationSet := migrate.MigrationSet{
			TableName: migrations.MigrationTableName,
		}
		ms := &migrate.MemoryMigrationSource{Migrations: migrations.All()}

		// It would be preferable to use migrate.MigrateDown() here, but that introduces
		// a circular dependency between testdb and datastore.
		n, err := migrationSet.ExecMax(db.DB, "postgres", ms, migrate.Down, down)
		require.NoError(tb, err)
		require.Equal(tb, down, n)
	}
}