PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Fix passing encypted password to .pgpass
Rostislav Dugin95c833b6193f
diff --git a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.goindex 7a350ef5..eed7a4d0 100644--- a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go@@ -81,6 +81,11 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(args := uc.buildPgDumpArgs(pg)+ decryptedPassword, err := uc.fieldEncryptor.Decrypt(db.ID, pg.Password)+ if err != nil {+ return nil, fmt.Errorf("failed to decrypt database password: %w", err)+ }+return uc.streamToStorage(ctx,backupID,@@ -92,7 +97,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(config.GetEnv().PostgresesInstallDir,),args,- pg.Password,+ decryptedPassword,storage,db,backupProgressListener,diff --git a/backend/internal/features/tests/postgresql_backup_restore_test.go b/backend/internal/features/tests/postgresql_backup_restore_test.goindex e39bbbe8..61c9ae22 100644--- a/backend/internal/features/tests/postgresql_backup_restore_test.go+++ b/backend/internal/features/tests/postgresql_backup_restore_test.go@@ -1,31 +1,36 @@package testsimport (- "context"+ "encoding/json""fmt"+ "net/http""os""path/filepath"- "postgresus-backend/internal/config"- "postgresus-backend/internal/features/backups/backups"- usecases_postgresql_backup "postgresus-backend/internal/features/backups/backups/usecases/postgresql"- backups_config "postgresus-backend/internal/features/backups/config"- "postgresus-backend/internal/features/databases"- pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"- "postgresus-backend/internal/features/intervals"- "postgresus-backend/internal/features/restores/models"- usecases_postgresql_restore "postgresus-backend/internal/features/restores/usecases/postgresql"- "postgresus-backend/internal/features/storages"- local_storage "postgresus-backend/internal/features/storages/models/local"- "postgresus-backend/internal/util/period"- "postgresus-backend/internal/util/tools""strconv""testing""time"+ "github.com/gin-gonic/gin""github.com/google/uuid""github.com/jmoiron/sqlx"_ "github.com/lib/pq""github.com/stretchr/testify/assert"++ "postgresus-backend/internal/config"+ "postgresus-backend/internal/features/backups/backups"+ backups_config "postgresus-backend/internal/features/backups/config"+ "postgresus-backend/internal/features/databases"+ pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"+ "postgresus-backend/internal/features/restores"+ restores_enums "postgresus-backend/internal/features/restores/enums"+ restores_models "postgresus-backend/internal/features/restores/models"+ "postgresus-backend/internal/features/storages"+ users_enums "postgresus-backend/internal/features/users/enums"+ users_testing "postgresus-backend/internal/features/users/testing"+ workspaces_controllers "postgresus-backend/internal/features/workspaces/controllers"+ workspaces_testing "postgresus-backend/internal/features/workspaces/testing"+ test_utils "postgresus-backend/internal/util/testing"+ "postgresus-backend/internal/util/tools")const createAndFillTableQuery = `@@ -61,7 +66,6 @@ type TestDataItem struct {CreatedAt time.Time `db:"created_at"`}-// Main test functions for each PostgreSQL versionfunc Test_BackupAndRestorePostgresql_RestoreIsSuccesful(t *testing.T) {env := config.GetEnv()cases := []struct {@@ -110,8 +114,7 @@ func Test_BackupAndRestorePostgresqlWithEncryption_RestoreIsSuccessful(t *testin}}-func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, port string) {- // Connect to pre-configured PostgreSQL container+func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {container, err := connectToPostgresContainer(pgVersion, port)assert.NoError(t, err)defer func() {@@ -123,109 +126,53 @@ func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, p_, err = container.DB.Exec(createAndFillTableQuery)assert.NoError(t, err)- // Prepare data for backup- backupID := uuid.New()- pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)-- backupDb := &databases.Database{- ID: uuid.New(),- Type: databases.DatabaseTypePostgres,- Name: "Test Database",- Postgresql: &pgtypes.PostgresqlDatabase{- Version: pgVersionEnum,- Host: container.Host,- Port: container.Port,- Username: container.Username,- Password: container.Password,- Database: &container.Database,- IsHttps: false,- },- }+ router := createTestRouter()+ user := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)- storageID := uuid.New()- backupConfig := &backups_config.BackupConfig{- DatabaseID: backupDb.ID,- IsBackupsEnabled: true,- StorePeriod: period.PeriodDay,- BackupInterval: &intervals.Interval{Interval: intervals.IntervalDaily},- StorageID: &storageID,- CpuCount: 1,- Encryption: backups_config.BackupEncryptionEncrypted,- }+ storage := storages.CreateTestStorage(workspace.ID)- storage := &storages.Storage{- WorkspaceID: uuid.New(),- Type: storages.StorageTypeLocal,- Name: "Test Storage",- LocalStorage: &local_storage.LocalStorage{},- }+ pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)+ database := createDatabaseViaAPI(+ t, router, "Test Database", workspace.ID,+ pgVersionEnum, container.Host, container.Port,+ container.Username, container.Password, container.Database,+ user.Token,+ )- // Make backup- progressTracker := func(completedMBs float64) {}- metadata, err := usecases_postgresql_backup.GetCreatePostgresqlBackupUsecase().Execute(- context.Background(),- backupID,- backupConfig,- backupDb,- storage,- progressTracker,+ enableBackupsViaAPI(+ t, router, database.ID, storage.ID,+ backups_config.BackupEncryptionNone, user.Token,)- assert.NoError(t, err)- assert.NotNil(t, metadata)- // Verify encryption metadata is set- assert.Equal(t, backups_config.BackupEncryptionEncrypted, metadata.Encryption)- assert.NotNil(t, metadata.EncryptionSalt)- assert.NotNil(t, metadata.EncryptionIV)+ createBackupViaAPI(t, router, database.ID, user.Token)- // Create new database- newDBName := "restoreddb_encrypted"+ backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)+ assert.Equal(t, backups.BackupStatusCompleted, backup.Status)++ newDBName := "restoreddb"_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))assert.NoError(t, err)_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))assert.NoError(t, err)- // Connect to the new databasenewDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",container.Host, container.Port, container.Username, container.Password, newDBName)newDB, err := sqlx.Connect("postgres", newDSN)assert.NoError(t, err)defer newDB.Close()- // Setup data for restore with encryption metadata- completedBackup := &backups.Backup{- ID: backupID,- DatabaseID: backupDb.ID,- StorageID: storage.ID,- Status: backups.BackupStatusCompleted,- CreatedAt: time.Now().UTC(),- EncryptionSalt: metadata.EncryptionSalt,- EncryptionIV: metadata.EncryptionIV,- Encryption: metadata.Encryption,- }-- restoreID := uuid.New()- restore := models.Restore{- ID: restoreID,- Backup: completedBackup,- Postgresql: &pgtypes.PostgresqlDatabase{- Version: pgVersionEnum,- Host: container.Host,- Port: container.Port,- Username: container.Username,- Password: container.Password,- Database: &newDBName,- IsHttps: false,- },- }+ createRestoreViaAPI(+ t, router, backup.ID, pgVersionEnum,+ container.Host, container.Port,+ container.Username, container.Password, newDBName,+ user.Token,+ )- // Restore the encrypted backup- restoreBackupUC := usecases_postgresql_restore.GetRestorePostgresqlBackupUsecase()- err = restoreBackupUC.Execute(backupDb, backupConfig, restore, completedBackup, storage)- assert.NoError(t, err)+ restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)+ assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)- // Verify restored table existsvar tableExists boolerr = newDB.Get(&tableExists,@@ -234,19 +181,25 @@ func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, passert.NoError(t, err)assert.True(t, tableExists, "Table 'test_data' should exist in restored database")- // Verify data integrityverifyDataIntegrity(t, container.DB, newDB)- // Clean up the backup file after the test- err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backupID.String()))+ err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))if err != nil {t.Logf("Warning: Failed to delete backup file: %v", err)}++ test_utils.MakeDeleteRequest(+ t,+ router,+ "/api/v1/databases/"+database.ID.String(),+ "Bearer "+user.Token,+ http.StatusNoContent,+ )+ storages.RemoveTestStorage(storage.ID)+ workspaces_testing.RemoveTestWorkspace(workspace, router)}-// Run a test for a specific PostgreSQL version-func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {- // Connect to pre-configured PostgreSQL container+func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, port string) {container, err := connectToPostgresContainer(pgVersion, port)assert.NoError(t, err)defer func() {@@ -258,99 +211,54 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {_, err = container.DB.Exec(createAndFillTableQuery)assert.NoError(t, err)- // Prepare data for backup- backupID := uuid.New()- pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)+ router := createTestRouter()+ user := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)- backupDb := &databases.Database{- ID: uuid.New(),- Type: databases.DatabaseTypePostgres,- Name: "Test Database",- Postgresql: &pgtypes.PostgresqlDatabase{- Version: pgVersionEnum,- Host: container.Host,- Port: container.Port,- Username: container.Username,- Password: container.Password,- Database: &container.Database,- IsHttps: false,- },- }+ storage := storages.CreateTestStorage(workspace.ID)- storageID := uuid.New()- backupConfig := &backups_config.BackupConfig{- DatabaseID: backupDb.ID,- IsBackupsEnabled: true,- StorePeriod: period.PeriodDay,- BackupInterval: &intervals.Interval{Interval: intervals.IntervalDaily},- StorageID: &storageID,- CpuCount: 1,- }-- storage := &storages.Storage{- WorkspaceID: uuid.New(),- Type: storages.StorageTypeLocal,- Name: "Test Storage",- LocalStorage: &local_storage.LocalStorage{},- }+ pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)+ database := createDatabaseViaAPI(+ t, router, "Test Database", workspace.ID,+ pgVersionEnum, container.Host, container.Port,+ container.Username, container.Password, container.Database,+ user.Token,+ )- // Make backup- progressTracker := func(completedMBs float64) {}- _, err = usecases_postgresql_backup.GetCreatePostgresqlBackupUsecase().Execute(- context.Background(),- backupID,- backupConfig,- backupDb,- storage,- progressTracker,+ enableBackupsViaAPI(+ t, router, database.ID, storage.ID,+ backups_config.BackupEncryptionEncrypted, user.Token,)- assert.NoError(t, err)- // Create new database- newDBName := "restoreddb"+ createBackupViaAPI(t, router, database.ID, user.Token)++ backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)+ assert.Equal(t, backups.BackupStatusCompleted, backup.Status)+ assert.Equal(t, backups_config.BackupEncryptionEncrypted, backup.Encryption)++ newDBName := "restoreddb_encrypted"_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))assert.NoError(t, err)_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))assert.NoError(t, err)- // Connect to the new databasenewDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",container.Host, container.Port, container.Username, container.Password, newDBName)newDB, err := sqlx.Connect("postgres", newDSN)assert.NoError(t, err)defer newDB.Close()- // Setup data for restore- completedBackup := &backups.Backup{- ID: backupID,- DatabaseID: backupDb.ID,- StorageID: storage.ID,- Status: backups.BackupStatusCompleted,- CreatedAt: time.Now().UTC(),- }-- restoreID := uuid.New()- restore := models.Restore{- ID: restoreID,- Backup: completedBackup,- Postgresql: &pgtypes.PostgresqlDatabase{- Version: pgVersionEnum,- Host: container.Host,- Port: container.Port,- Username: container.Username,- Password: container.Password,- Database: &newDBName,- IsHttps: false,- },- }+ createRestoreViaAPI(+ t, router, backup.ID, pgVersionEnum,+ container.Host, container.Port,+ container.Username, container.Password, newDBName,+ user.Token,+ )- // Restore the backup- restoreBackupUC := usecases_postgresql_restore.GetRestorePostgresqlBackupUsecase()- err = restoreBackupUC.Execute(backupDb, backupConfig, restore, completedBackup, storage)- assert.NoError(t, err)+ restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)+ assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)- // Verify restored table existsvar tableExists boolerr = newDB.Get(&tableExists,@@ -359,17 +267,243 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {assert.NoError(t, err)assert.True(t, tableExists, "Table 'test_data' should exist in restored database")- // Verify data integrityverifyDataIntegrity(t, container.DB, newDB)- // Clean up the backup file after the test- err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backupID.String()))+ err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))if err != nil {t.Logf("Warning: Failed to delete backup file: %v", err)}++ test_utils.MakeDeleteRequest(+ t,+ router,+ "/api/v1/databases/"+database.ID.String(),+ "Bearer "+user.Token,+ http.StatusNoContent,+ )+ storages.RemoveTestStorage(storage.ID)+ workspaces_testing.RemoveTestWorkspace(workspace, router)+}++func createTestRouter() *gin.Engine {+ router := workspaces_testing.CreateTestRouter(+ workspaces_controllers.GetWorkspaceController(),+ workspaces_controllers.GetMembershipController(),+ databases.GetDatabaseController(),+ backups_config.GetBackupConfigController(),+ backups.GetBackupController(),+ restores.GetRestoreController(),+ )+ return router+}++func waitForBackupCompletion(+ t *testing.T,+ router *gin.Engine,+ databaseID uuid.UUID,+ token string,+ timeout time.Duration,+) *backups.Backup {+ startTime := time.Now()+ pollInterval := 500 * time.Millisecond++ for {+ if time.Since(startTime) > timeout {+ t.Fatalf("Timeout waiting for backup completion after %v", timeout)+ }++ var response backups.GetBackupsResponse+ test_utils.MakeGetRequestAndUnmarshal(+ t,+ router,+ fmt.Sprintf("/api/v1/backups?database_id=%s&limit=1", databaseID.String()),+ "Bearer "+token,+ http.StatusOK,+ &response,+ )++ if len(response.Backups) > 0 {+ backup := response.Backups[0]+ if backup.Status == backups.BackupStatusCompleted {+ return backup+ }+ if backup.Status == backups.BackupStatusFailed {+ t.Fatalf("Backup failed: %v", backup.FailMessage)+ }+ }++ time.Sleep(pollInterval)+ }+}++func waitForRestoreCompletion(+ t *testing.T,+ router *gin.Engine,+ backupID uuid.UUID,+ token string,+ timeout time.Duration,+) *restores_models.Restore {+ startTime := time.Now()+ pollInterval := 500 * time.Millisecond++ for {+ if time.Since(startTime) > timeout {+ t.Fatalf("Timeout waiting for restore completion after %v", timeout)+ }++ var restores []*restores_models.Restore+ test_utils.MakeGetRequestAndUnmarshal(+ t,+ router,+ fmt.Sprintf("/api/v1/restores/%s", backupID.String()),+ "Bearer "+token,+ http.StatusOK,+ &restores,+ )++ for _, restore := range restores {+ if restore.Status == restores_enums.RestoreStatusCompleted {+ return restore+ }+ if restore.Status == restores_enums.RestoreStatusFailed {+ t.Fatalf("Restore failed: %v", restore.FailMessage)+ }+ }++ time.Sleep(pollInterval)+ }+}++func createDatabaseViaAPI(+ t *testing.T,+ router *gin.Engine,+ name string,+ workspaceID uuid.UUID,+ pgVersion tools.PostgresqlVersion,+ host string,+ port int,+ username string,+ password string,+ database string,+ token string,+) *databases.Database {+ request := databases.Database{+ Name: name,+ WorkspaceID: &workspaceID,+ Type: databases.DatabaseTypePostgres,+ Postgresql: &pgtypes.PostgresqlDatabase{+ Version: pgVersion,+ Host: host,+ Port: port,+ Username: username,+ Password: password,+ Database: &database,+ },+ }++ w := workspaces_testing.MakeAPIRequest(+ router,+ "POST",+ "/api/v1/databases/create",+ "Bearer "+token,+ request,+ )++ if w.Code != http.StatusCreated {+ t.Fatalf("Failed to create database. Status: %d, Body: %s", w.Code, w.Body.String())+ }++ var createdDatabase databases.Database+ if err := json.Unmarshal(w.Body.Bytes(), &createdDatabase); err != nil {+ t.Fatalf("Failed to unmarshal database response: %v", err)+ }++ return &createdDatabase+}++func enableBackupsViaAPI(+ t *testing.T,+ router *gin.Engine,+ databaseID uuid.UUID,+ storageID uuid.UUID,+ encryption backups_config.BackupEncryption,+ token string,+) {+ var backupConfig backups_config.BackupConfig+ test_utils.MakeGetRequestAndUnmarshal(+ t,+ router,+ fmt.Sprintf("/api/v1/backup-configs/database/%s", databaseID.String()),+ "Bearer "+token,+ http.StatusOK,+ &backupConfig,+ )++ storage := &storages.Storage{ID: storageID}+ backupConfig.IsBackupsEnabled = true+ backupConfig.Storage = storage+ backupConfig.Encryption = encryption++ test_utils.MakePostRequest(+ t,+ router,+ "/api/v1/backup-configs/save",+ "Bearer "+token,+ backupConfig,+ http.StatusOK,+ )+}++func createBackupViaAPI(+ t *testing.T,+ router *gin.Engine,+ databaseID uuid.UUID,+ token string,+) {+ request := backups.MakeBackupRequest{DatabaseID: databaseID}+ test_utils.MakePostRequest(+ t,+ router,+ "/api/v1/backups",+ "Bearer "+token,+ request,+ http.StatusOK,+ )+}++func createRestoreViaAPI(+ t *testing.T,+ router *gin.Engine,+ backupID uuid.UUID,+ pgVersion tools.PostgresqlVersion,+ host string,+ port int,+ username string,+ password string,+ database string,+ token string,+) {+ request := restores.RestoreBackupRequest{+ PostgresqlDatabase: &pgtypes.PostgresqlDatabase{+ Version: pgVersion,+ Host: host,+ Port: port,+ Username: username,+ Password: password,+ Database: &database,+ },+ }++ test_utils.MakePostRequest(+ t,+ router,+ fmt.Sprintf("/api/v1/restores/%s/restore", backupID.String()),+ "Bearer "+token,+ request,+ http.StatusOK,+ )}-// verifyDataIntegrity compares data in the original and restored databasesfunc verifyDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB) {var originalData []TestDataItemvar restoredData []TestDataItem@@ -382,7 +516,6 @@ func verifyDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB)assert.Equal(t, len(originalData), len(restoredData), "Should have same number of rows")- // Only compare data if both slices have elements (to avoid panic)if len(originalData) > 0 && len(restoredData) > 0 {for i := range originalData {assert.Equal(t, originalData[i].ID, restoredData[i].ID, "ID should match")diff --git a/frontend/src/widgets/main/MainScreenComponent.tsx b/frontend/src/widgets/main/MainScreenComponent.tsxindex 2ded3a0d..168e7881 100644--- a/frontend/src/widgets/main/MainScreenComponent.tsx+++ b/frontend/src/widgets/main/MainScreenComponent.tsx@@ -140,6 +140,23 @@ export const MainScreenComponent = () => {</div><div className="ml-auto flex items-center gap-5">+ <a+ className="!text-black hover:opacity-80"+ href="https://postgresus.com/installation"+ target="_blank"+ rel="noreferrer"+ >+ Docs+ </a>++ <a+ className="!text-black hover:opacity-80"+ href="https://postgresus.com/contribute"+ target="_blank"+ rel="noreferrer"+ >+ Contribute+ </a><aclassName="!text-black hover:opacity-80"href="https://t.me/postgresus_community"