Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

20.4 KB
Raw Blame
package main

import (
	"bytes"
	"context"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/google/uuid"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"databasus-backend/internal/config"
	audit_logs "databasus-backend/internal/features/audit_logs"
	backuping_logical "databasus-backend/internal/features/backups/backups/backuping/logical"
	backuping_physical "databasus-backend/internal/features/backups/backups/backuping/physical"
	backups_controllers_logical "databasus-backend/internal/features/backups/backups/controllers/logical"
	backups_controllers_physical "databasus-backend/internal/features/backups/backups/controllers/physical"
	backups_dto_logical "databasus-backend/internal/features/backups/backups/dto/logical"
	backups_services "databasus-backend/internal/features/backups/backups/services"
	backups_config_logical "databasus-backend/internal/features/backups/config/logical"
	backups_config_physical "databasus-backend/internal/features/backups/config/physical"
	"databasus-backend/internal/features/databases"
	"databasus-backend/internal/features/email"
	"databasus-backend/internal/features/notifiers"
	"databasus-backend/internal/features/storages"
	local_storage "databasus-backend/internal/features/storages/models/local"
	system_healthcheck "databasus-backend/internal/features/system/healthcheck"
	users_controllers "databasus-backend/internal/features/users/controllers"
	users_dto "databasus-backend/internal/features/users/dto"
	users_enums "databasus-backend/internal/features/users/enums"
	users_middleware "databasus-backend/internal/features/users/middleware"
	users_services "databasus-backend/internal/features/users/services"
	users_testing "databasus-backend/internal/features/users/testing"
	workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
	workspaces_dto "databasus-backend/internal/features/workspaces/dto"
	workspaces_errors "databasus-backend/internal/features/workspaces/errors"
	test_utils "databasus-backend/internal/util/testing"
)

const (
	seedPassword  = "Q1234567"
	otherPassword = "Other-password-1"

	firstBackupTimeout = 2 * time.Minute
)

func Test_Seed_WhenAccountIsNew_CreatesWorkspaceStorageAndDatabasesWithBackups(t *testing.T) {
	router, params := setUpSeedTest(t)

	exitCode, output := runSeedForTest(t, params)
	require.Equal(t, 0, exitCode, output)

	token := signInForTest(t, router, params.Email, seedPassword)
	workspaceID := findWorkspaceID(t, router, token, params.WorkspaceName)
	require.NotEqual(t, uuid.Nil, workspaceID, "workspace %q was not created", params.WorkspaceName)

	storageID := findStorageID(t, router, token, workspaceID, params.StorageName)
	require.NotEqual(t, uuid.Nil, storageID, "storage %q was not created", params.StorageName)

	logicalDatabaseID := findDatabaseID(t, router, token, workspaceID, params.LogicalDatabaseName)
	walDatabaseID := findDatabaseID(t, router, token, workspaceID, params.WalDatabaseName)
	require.NotEqual(t, uuid.Nil, logicalDatabaseID)
	require.NotEqual(t, uuid.Nil, walDatabaseID)

	var logicalConfig backups_config_logical.LogicalBackupConfig
	test_utils.MakeGetRequestAndUnmarshal(t, router,
		"/api/v1/backup-configs/database/"+logicalDatabaseID.String(),
		"Bearer "+token, http.StatusOK, &logicalConfig)

	assert.True(t, logicalConfig.IsBackupsEnabled)
	assert.Equal(t, storageID, *logicalConfig.StorageID)
	assert.Equal(t, backups_config_logical.RetentionPolicyTypeCount, logicalConfig.RetentionPolicyType)
	assert.Equal(t, 3, logicalConfig.RetentionCount)

	var logicalBackups backups_dto_logical.GetBackupsResponse
	test_utils.MakeGetRequestAndUnmarshal(t, router,
		"/api/v1/backups?database_id="+logicalDatabaseID.String(),
		"Bearer "+token, http.StatusOK, &logicalBackups)
	assert.NotEmpty(t, logicalBackups.Backups, "the first logical backup was not started")

	var walConfig backups_config_physical.PhysicalBackupConfig
	test_utils.MakeGetRequestAndUnmarshal(t, router,
		"/api/v1/backup-configs/physical/database/"+walDatabaseID.String(),
		"Bearer "+token, http.StatusOK, &walConfig)

	assert.True(t, walConfig.IsBackupsEnabled)
	assert.Equal(t, storageID, *walConfig.StorageID)
	assert.Equal(t, backups_config_physical.RetentionChains, walConfig.Retention)
	assert.Equal(t, 2, walConfig.ChainsRetention.Count)

	storedWalConfig, err := backups_config_physical.GetBackupConfigService().GetBackupConfigByDbId(walDatabaseID)
	require.NoError(t, err)
	assert.NotNil(t, storedWalConfig.ForceFullRequestedAt, "the first WAL-streaming backup was not requested")
}

func Test_Seed_WhenRunTwice_CreatesNothingAndKeepsEditedDatabase(t *testing.T) {
	router, params := setUpSeedTest(t)

	exitCode, output := runSeedForTest(t, params)
	require.Equal(t, 0, exitCode, output)

	token := signInForTest(t, router, params.Email, seedPassword)
	workspaceID := findWorkspaceID(t, router, token, params.WorkspaceName)
	logicalDatabaseID := findDatabaseID(t, router, token, workspaceID, params.LogicalDatabaseName)
	waitForLogicalBackups(t, logicalDatabaseID)

	var editedConfig backups_config_logical.LogicalBackupConfig
	test_utils.MakeGetRequestAndUnmarshal(t, router,
		"/api/v1/backup-configs/database/"+logicalDatabaseID.String(),
		"Bearer "+token, http.StatusOK, &editedConfig)

	editedConfig.RetentionCount = 9
	editedConfig.IsBackupsEnabled = false
	test_utils.MakePostRequest(t, router, "/api/v1/backup-configs/save",
		"Bearer "+token, editedConfig, http.StatusOK)

	itemsBefore := listSeededItems(t, router, token)

	exitCode, output = runSeedForTest(t, params)
	require.Equal(t, 0, exitCode, output)

	assert.Equal(t, itemsBefore, listSeededItems(t, router, token))

	var configAfter backups_config_logical.LogicalBackupConfig
	test_utils.MakeGetRequestAndUnmarshal(t, router,
		"/api/v1/backup-configs/database/"+logicalDatabaseID.String(),
		"Bearer "+token, http.StatusOK, &configAfter)

	assert.Equal(t, 9, configAfter.RetentionCount)
	assert.False(t, configAfter.IsBackupsEnabled)
}

func Test_Seed_WhenSeededDatabaseWasDeleted_RecreatesOnlyThatDatabase(t *testing.T) {
	router, params := setUpSeedTest(t)

	exitCode, output := runSeedForTest(t, params)
	require.Equal(t, 0, exitCode, output)

	token := signInForTest(t, router, params.Email, seedPassword)
	workspaceID := findWorkspaceID(t, router, token, params.WorkspaceName)
	storageID := findStorageID(t, router, token, workspaceID, params.StorageName)
	logicalDatabaseID := findDatabaseID(t, router, token, workspaceID, params.LogicalDatabaseName)
	deletedWalDatabaseID := findDatabaseID(t, router, token, workspaceID, params.WalDatabaseName)

	test_utils.MakeDeleteRequest(t, router, "/api/v1/databases/"+deletedWalDatabaseID.String(),
		"Bearer "+token, http.StatusNoContent)

	exitCode, output = runSeedForTest(t, params)
	require.Equal(t, 0, exitCode, output)

	recreatedWalDatabaseID := findDatabaseID(t, router, token, workspaceID, params.WalDatabaseName)
	require.NotEqual(t, uuid.Nil, recreatedWalDatabaseID)
	assert.NotEqual(t, deletedWalDatabaseID, recreatedWalDatabaseID)

	var walConfig backups_config_physical.PhysicalBackupConfig
	test_utils.MakeGetRequestAndUnmarshal(t, router,
		"/api/v1/backup-configs/physical/database/"+recreatedWalDatabaseID.String(),
		"Bearer "+token, http.StatusOK, &walConfig)
	assert.True(t, walConfig.IsBackupsEnabled)

	assert.Equal(t, workspaceID, findWorkspaceID(t, router, token, params.WorkspaceName))
	assert.Equal(t, storageID, findStorageID(t, router, token, workspaceID, params.StorageName))
	assert.Equal(t, logicalDatabaseID, findDatabaseID(t, router, token, workspaceID, params.LogicalDatabaseName))
	assert.Len(t, listSeededItems(t, router, token).databaseIDs, 2)
}

func Test_Seed_WhenAccountHasAnotherPassword_FailsAndLeavesAccountUnchanged(t *testing.T) {
	router, params := setUpSeedTest(t)

	signUpForTest(t, router, params.Email, otherPassword)
	token := signInForTest(t, router, params.Email, otherPassword)
	profileBefore := getProfile(t, router, token)

	exitCode, output := runSeedForTest(t, params)

	assert.Equal(t, 1, exitCode)
	assert.Contains(t, output, params.Email)
	assert.Contains(t, output, "password is incorrect")

	tokenAfter := signInForTest(t, router, params.Email, otherPassword)
	assert.Equal(t, profileBefore, getProfile(t, router, tokenAfter))
	assert.Empty(t, listSeededItems(t, router, tokenAfter).workspaceIDs)
}

func Test_Seed_WhenAnotherAccountExists_LeavesThatAccountUnchanged(t *testing.T) {
	router, params := setUpSeedTest(t)

	otherEmail := "seed-other-" + uuid.NewString() + "@example.com"
	signUpForTest(t, router, otherEmail, otherPassword)
	t.Cleanup(func() { removeAccountWithItems(t, router, otherEmail) })

	otherProfileBefore := getProfile(t, router, signInForTest(t, router, otherEmail, otherPassword))

	exitCode, output := runSeedForTest(t, params)
	require.Equal(t, 0, exitCode, output)

	otherToken := signInForTest(t, router, otherEmail, otherPassword)
	assert.Equal(t, otherProfileBefore, getProfile(t, router, otherToken))
	assert.Empty(t, listSeededItems(t, router, otherToken).workspaceIDs)
}

func Test_Seed_WhenTwoFactorSignInIsRequired_FailsAndCreatesNothing(t *testing.T) {
	router, params := setUpSeedTest(t)

	token := signUpForTest(t, router, params.Email, seedPassword)

	users_services.GetUserService().SetEmailSender(users_testing.NewMockEmailSender())
	users_testing.EnableTwoFactorAuth(t.Context())
	t.Cleanup(func() {
		users_services.GetUserService().SetEmailSender(email.GetEmailSMTPSender())
		users_testing.ResetSettingsToDefaults(context.Background())
	})

	exitCode, output := runSeedForTest(t, params)

	assert.Equal(t, 1, exitCode)
	assert.Contains(t, output, params.Email)
	assert.Contains(t, output, "two-factor sign-in")
	assert.Empty(t, listSeededItems(t, router, token).workspaceIDs)
}

func Test_Seed_WhenMemberMayNotCreateWorkspaces_FailsNamingWorkspace(t *testing.T) {
	router, params := setUpSeedTest(t)

	token := signUpForTest(t, router, params.Email, seedPassword)
	require.Equal(t, users_enums.UserRoleMember, getProfile(t, router, token).Role)

	var existingWorkspace workspaces_dto.WorkspaceResponseDTO
	test_utils.MakePostRequestAndUnmarshal(t, router, "/api/v1/workspaces", "Bearer "+token,
		workspaces_dto.CreateWorkspaceRequestDTO{Name: "Existing"}, http.StatusOK, &existingWorkspace)

	test_utils.MakePostRequest(t, router, "/api/v1/storages", "Bearer "+token, storages.Storage{
		WorkspaceID:  existingWorkspace.ID,
		Type:         storages.StorageTypeLocal,
		Name:         "Existing storage",
		LocalStorage: &local_storage.LocalStorage{},
	}, http.StatusOK)

	workspacesBefore := listWorkspaces(t, router, token)
	storagesBefore := listStorages(t, router, token, existingWorkspace.ID)

	users_testing.DisableMemberWorkspaceCreation(t.Context())
	t.Cleanup(func() { users_testing.ResetSettingsToDefaults(context.Background()) })

	exitCode, output := runSeedForTest(t, params)

	assert.Equal(t, 1, exitCode)
	assert.Contains(t, output, params.WorkspaceName)
	assert.Contains(t, output, workspaces_errors.ErrInsufficientPermissionsToCreateWorkspaces.Error())

	assert.Equal(t, workspacesBefore, listWorkspaces(t, router, token))
	assert.Equal(t, storagesBefore, listStorages(t, router, token, existingWorkspace.ID))
	assert.Equal(t, uuid.Nil, findWorkspaceID(t, router, token, params.WorkspaceName))
}

func Test_Seed_WhenBackendDoesNotAnswer_FailsWithJustRunHint(t *testing.T) {
	stoppedServer := httptest.NewServer(http.NotFoundHandler())
	stoppedServer.Close()

	params := buildTestSeedParams(stoppedServer.URL)

	exitCode, output := runSeedForTest(t, params)

	assert.Equal(t, 1, exitCode)
	assert.Contains(t, output, "just run")
}

// A full disk is unsafe to reproduce, so a stub stands in for the backend's health check.
func Test_Seed_WhenBackendReportsItselfUnhealthy_FailsWithReasonAndSendsNothingElse(t *testing.T) {
	const unhealthyReason = "more than 95% of the disk is used"

	var receivedPathsMutex sync.Mutex
	var receivedPaths []string

	stubServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		receivedPathsMutex.Lock()
		receivedPaths = append(receivedPaths, request.URL.Path)
		receivedPathsMutex.Unlock()

		writer.Header().Set("Content-Type", "application/json")
		writer.WriteHeader(http.StatusServiceUnavailable)
		_, _ = writer.Write([]byte(`{"status":"` + unhealthyReason + `"}`))
	}))
	t.Cleanup(stubServer.Close)

	exitCode, output := runSeedForTest(t, buildTestSeedParams(stubServer.URL))

	assert.Equal(t, 1, exitCode)
	assert.Contains(t, output, unhealthyReason)

	receivedPathsMutex.Lock()
	defer receivedPathsMutex.Unlock()
	assert.Equal(t, []string{"/api/v1/system/health"}, receivedPaths)
}

type seededItems struct {
	workspaceIDs []uuid.UUID
	storageIDs   []uuid.UUID
	databaseIDs  []uuid.UUID
}

// Every test claims the instance with an account of its own first, so the seeded account
// registers as a member, the way it does on a dev instance someone already uses.
func setUpSeedTest(t *testing.T) (*gin.Engine, seedParams) {
	t.Helper()

	router := newSeedTestRouter()
	server := httptest.NewServer(router)
	t.Cleanup(server.Close)

	users_testing.ResetSettingsToDefaults(t.Context())

	instanceOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
	t.Cleanup(func() { users_testing.DeleteTestUser(context.Background(), instanceOwner.UserID) })

	params := buildTestSeedParams(server.URL)
	t.Cleanup(func() { removeAccountWithItems(t, router, params.Email) })

	return router, params
}

func newSeedTestRouter() *gin.Engine {
	gin.SetMode(gin.TestMode)
	router := gin.New()

	v1 := router.Group("/api/v1")

	userController := users_controllers.GetUserController()
	userController.RegisterRoutes(v1)
	system_healthcheck.GetHealthcheckController().RegisterRoutes(v1)

	protected := v1.Group("")
	protected.Use(users_middleware.AuthMiddleware(users_services.GetUserService()))

	userController.RegisterProtectedRoutes(protected)
	workspaces_controllers.GetWorkspaceController().RegisterRoutes(protected)
	storages.GetStorageController().RegisterRoutes(protected)
	databases.GetDatabaseController().RegisterRoutes(protected)
	backups_config_logical.GetBackupConfigController().RegisterRoutes(protected)
	backups_config_physical.GetBackupConfigController().RegisterRoutes(protected)
	backups_controllers_logical.GetBackupController().RegisterRoutes(protected)
	backups_controllers_physical.GetPhysicalBackupController().RegisterRoutes(protected)

	databases.SetupDependencies()
	backups_services.SetupDependencies()
	audit_logs.SetupDependencies()
	notifiers.SetupDependencies()
	storages.SetupDependencies()
	backups_config_logical.SetupDependencies()
	backups_config_physical.SetupDependencies()
	backuping_physical.SetupDependencies()

	return router
}

func buildTestSeedParams(baseURL string) seedParams {
	env := config.GetEnv()

	logicalPort, err := strconv.Atoi(env.TestLogicalPostgres16Port)
	if err != nil {
		panic(err)
	}

	physicalPort, err := strconv.Atoi(env.TestPhysicalPostgres18Port)
	if err != nil {
		panic(err)
	}

	return seedParams{
		BaseURL:             baseURL,
		Email:               "seed-" + uuid.NewString() + "@example.com",
		Password:            seedPassword,
		WorkspaceName:       "Dev",
		StorageName:         "Local",
		LogicalDatabaseName: "Logical PG16",
		LogicalSource: postgresSource{
			Host:     env.TestLocalhost,
			Port:     logicalPort,
			Username: "testuser",
			Password: "testpassword",
			Database: "testdb",
		},
		WalDatabaseName: "WAL PG18",
		WalSource: postgresSource{
			Host:     env.TestLocalhost,
			Port:     physicalPort,
			Username: "testuser",
			Password: "testpassword",
		},
	}
}

func runSeedForTest(t *testing.T, params seedParams) (int, string) {
	t.Helper()

	var output bytes.Buffer
	exitCode := runSeed(t.Context(), &http.Client{Timeout: requestTimeout}, params,
		slog.New(slog.NewTextHandler(&output, nil)))

	return exitCode, output.String()
}

func signUpForTest(t *testing.T, router *gin.Engine, email, password string) string {
	t.Helper()

	var signUp users_dto.SignInResponseDTO
	test_utils.MakePostRequestAndUnmarshal(t, router, "/api/v1/users/signup", "",
		users_dto.SignUpRequestDTO{Email: email, Password: password, Name: "Seed Test"},
		http.StatusOK, &signUp)

	return signUp.Token
}

func signInForTest(t *testing.T, router *gin.Engine, email, password string) string {
	t.Helper()

	var signIn users_dto.SignInResponseDTO
	test_utils.MakePostRequestAndUnmarshal(t, router, "/api/v1/users/signin", "",
		users_dto.SignInRequestDTO{Email: email, Password: password}, http.StatusOK, &signIn)
	require.NotEmpty(t, signIn.Token)

	return signIn.Token
}

func getProfile(t *testing.T, router *gin.Engine, token string) users_dto.UserProfileResponseDTO {
	t.Helper()

	var profile users_dto.UserProfileResponseDTO
	test_utils.MakeGetRequestAndUnmarshal(t, router, "/api/v1/users/me", "Bearer "+token,
		http.StatusOK, &profile)

	return profile
}

func listWorkspaces(t *testing.T, router *gin.Engine, token string) []workspaces_dto.WorkspaceResponseDTO {
	t.Helper()

	var listedWorkspaces workspaces_dto.ListWorkspacesResponseDTO
	test_utils.MakeGetRequestAndUnmarshal(t, router, "/api/v1/workspaces", "Bearer "+token,
		http.StatusOK, &listedWorkspaces)

	return listedWorkspaces.Workspaces
}

func findWorkspaceID(t *testing.T, router *gin.Engine, token, name string) uuid.UUID {
	t.Helper()

	for _, workspace := range listWorkspaces(t, router, token) {
		if workspace.Name == name {
			return workspace.ID
		}
	}

	return uuid.Nil
}

func findStorageID(t *testing.T, router *gin.Engine, token string, workspaceID uuid.UUID, name string) uuid.UUID {
	t.Helper()

	for _, storage := range listStorages(t, router, token, workspaceID) {
		if storage.Name == name {
			return storage.ID
		}
	}

	return uuid.Nil
}

func findDatabaseID(t *testing.T, router *gin.Engine, token string, workspaceID uuid.UUID, name string) uuid.UUID {
	t.Helper()

	for _, database := range listDatabases(t, router, token, workspaceID) {
		if database.Name == name {
			return database.ID
		}
	}

	return uuid.Nil
}

func listStorages(t *testing.T, router *gin.Engine, token string, workspaceID uuid.UUID) []storages.Storage {
	t.Helper()

	var listedStorages []storages.Storage
	test_utils.MakeGetRequestAndUnmarshal(t, router, "/api/v1/storages?workspace_id="+workspaceID.String(),
		"Bearer "+token, http.StatusOK, &listedStorages)

	return listedStorages
}

func listDatabases(t *testing.T, router *gin.Engine, token string, workspaceID uuid.UUID) []databases.Database {
	t.Helper()

	var listedDatabases []databases.Database
	test_utils.MakeGetRequestAndUnmarshal(t, router, "/api/v1/databases?workspace_id="+workspaceID.String(),
		"Bearer "+token, http.StatusOK, &listedDatabases)

	return listedDatabases
}

func listSeededItems(t *testing.T, router *gin.Engine, token string) seededItems {
	t.Helper()

	var items seededItems

	for _, workspace := range listWorkspaces(t, router, token) {
		items.workspaceIDs = append(items.workspaceIDs, workspace.ID)

		for _, storage := range listStorages(t, router, token, workspace.ID) {
			items.storageIDs = append(items.storageIDs, storage.ID)
		}

		for _, database := range listDatabases(t, router, token, workspace.ID) {
			items.databaseIDs = append(items.databaseIDs, database.ID)
		}
	}

	return items
}

// A logical backup the seed started still writes its file, and deleting its database before it
// finishes would leave the file behind.
func waitForLogicalBackups(t *testing.T, databaseID uuid.UUID) {
	t.Helper()

	backuping_logical.WaitForBackupCompletion(t, databaseID, 0, firstBackupTimeout)
}

func removeAccountWithItems(t *testing.T, router *gin.Engine, email string) {
	t.Helper()

	ctx := context.Background()

	user, err := users_services.GetUserService().GetUserByEmail(ctx, email)
	if err != nil || user == nil {
		return
	}

	access, err := users_services.GetUserService().GenerateAccessToken(ctx, user)
	require.NoError(t, err)

	token := access.Token
	items := listSeededItems(t, router, token)

	for _, workspaceID := range items.workspaceIDs {
		for _, database := range listDatabases(t, router, token, workspaceID) {
			if database.Type == databases.DatabaseTypePostgresLogical {
				waitForLogicalBackups(t, database.ID)
			}

			test_utils.MakeDeleteRequest(t, router, "/api/v1/databases/"+database.ID.String(),
				"Bearer "+token, http.StatusNoContent)
		}

		for _, storage := range listStorages(t, router, token, workspaceID) {
			test_utils.MakeDeleteRequest(t, router, "/api/v1/storages/"+storage.ID.String(),
				"Bearer "+token, http.StatusOK)
		}

		test_utils.MakeDeleteRequest(t, router, "/api/v1/workspaces/"+workspaceID.String(),
			"Bearer "+token, http.StatusOK)
	}

	users_testing.DeleteTestUser(ctx, user.ID)
}