PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): prevent hang from SaveFile failur if error happens early
Rostislav Dugine465205fcb9d
diff --git a/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.goindex 515f785e..9f192109 100644--- a/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go@@ -228,6 +228,10 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(backup.FileName,storageReader,)+ if saveErr != nil {+ _ = storageReader.CloseWithError(saveErr)+ cancel()+ }saveErrCh <- saveErr}()@@ -252,6 +256,17 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(bytesWritten := <-bytesWrittenChwaitErr := cmd.Wait()+ select {+ case earlySaveErr := <-saveErrCh:+ if earlySaveErr != nil {+ _ = zstdWriter.Close()+ _ = uc.closeWriters(encryptionWriter, storageWriter)+ return nil, fmt.Errorf("save to storage: %w", earlySaveErr)+ }+ saveErrCh <- nil+ default:+ }+select {case <-ctx.Done():uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)diff --git a/backend/internal/features/backups/backups/usecases/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/mongodb/create_backup_uc.goindex 003a945f..c512a04b 100644--- a/backend/internal/features/backups/backups/usecases/mongodb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/mongodb/create_backup_uc.go@@ -190,6 +190,10 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(backup.FileName,storageReader,)+ if saveErr != nil {+ _ = storageReader.CloseWithError(saveErr)+ cancel()+ }saveErrCh <- saveErr}()@@ -214,6 +218,16 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(bytesWritten := <-bytesWrittenChwaitErr := cmd.Wait()+ select {+ case earlySaveErr := <-saveErrCh:+ if earlySaveErr != nil {+ _ = uc.closeWriters(encryptionWriter, storageWriter)+ return nil, fmt.Errorf("save to storage: %w", earlySaveErr)+ }+ saveErrCh <- nil+ default:+ }+select {case <-ctx.Done():uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)diff --git a/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.goindex e5770c00..555eec69 100644--- a/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go@@ -247,6 +247,10 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(backup.FileName,storageReader,)+ if saveErr != nil {+ _ = storageReader.CloseWithError(saveErr)+ cancel()+ }saveErrCh <- saveErr}()@@ -271,6 +275,17 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(bytesWritten := <-bytesWrittenChwaitErr := cmd.Wait()+ select {+ case earlySaveErr := <-saveErrCh:+ if earlySaveErr != nil {+ _ = zstdWriter.Close()+ _ = uc.closeWriters(encryptionWriter, storageWriter)+ return nil, fmt.Errorf("save to storage: %w", earlySaveErr)+ }+ saveErrCh <- nil+ default:+ }+select {case <-ctx.Done():uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)diff --git a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.goindex 52062395..19556425 100644--- a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go@@ -192,6 +192,10 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(backup.FileName,storageReader,)+ if saveErr != nil {+ _ = storageReader.CloseWithError(saveErr)+ cancel()+ }saveErrCh <- saveErr}()@@ -218,6 +222,16 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(bytesWritten := <-bytesWrittenChwaitErr := cmd.Wait()+ select {+ case earlySaveErr := <-saveErrCh:+ if earlySaveErr != nil {+ _ = uc.closeWriters(encryptionWriter, storageWriter)+ return nil, fmt.Errorf("save to storage: %w", earlySaveErr)+ }+ saveErrCh <- nil+ default:+ }+select {case <-ctx.Done():uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)diff --git a/backend/internal/features/storages/testing.go b/backend/internal/features/storages/testing.goindex 768e2875..67768241 100644--- a/backend/internal/features/storages/testing.go+++ b/backend/internal/features/storages/testing.go@@ -4,6 +4,8 @@ import ("github.com/google/uuid"local_storage "databasus-backend/internal/features/storages/models/local"+ s3_storage "databasus-backend/internal/features/storages/models/s3"+ "databasus-backend/internal/util/encryption")func CreateTestStorage(workspaceID uuid.UUID) *Storage {@@ -22,6 +24,40 @@ func CreateTestStorage(workspaceID uuid.UUID) *Storage {return storage}+func CreateTestFlakyS3Storage(workspaceID uuid.UUID, endpoint string) *Storage {+ encryptor := encryption.GetFieldEncryptor()++ accessKey, err := encryptor.Encrypt("issue-582-access")+ if err != nil {+ panic(err)+ }++ secretKey, err := encryptor.Encrypt("issue-582-secret")+ if err != nil {+ panic(err)+ }++ storage := &Storage{+ WorkspaceID: workspaceID,+ Type: StorageTypeS3,+ Name: "Flaky S3 " + uuid.New().String(),+ S3Storage: &s3_storage.S3Storage{+ S3Bucket: "issue-582-no-such-bucket-" + uuid.New().String(),+ S3Region: "us-east-1",+ S3AccessKey: accessKey,+ S3SecretKey: secretKey,+ S3Endpoint: endpoint,+ },+ }++ saved, err := storageRepository.Save(storage)+ if err != nil {+ panic(err)+ }++ return saved+}+func RemoveTestStorage(id uuid.UUID) {storage, err := storageRepository.FindByID(id)if err != nil {diff --git a/backend/internal/features/tests/backup_hang_edge_cases_test.go b/backend/internal/features/tests/backup_hang_edge_cases_test.gonew file mode 100644index 00000000..6c247c83--- /dev/null+++ b/backend/internal/features/tests/backup_hang_edge_cases_test.go@@ -0,0 +1,223 @@+package tests++import (+ "fmt"+ "net/http"+ "testing"+ "time"++ "github.com/gin-gonic/gin"+ _ "github.com/go-sql-driver/mysql"+ "github.com/google/uuid"+ "github.com/stretchr/testify/require"++ "databasus-backend/internal/config"+ backups_core "databasus-backend/internal/features/backups/backups/core"+ backups_dto "databasus-backend/internal/features/backups/backups/dto"+ backups_config "databasus-backend/internal/features/backups/config"+ "databasus-backend/internal/features/storages"+ users_enums "databasus-backend/internal/features/users/enums"+ users_testing "databasus-backend/internal/features/users/testing"+ workspaces_testing "databasus-backend/internal/features/workspaces/testing"+ test_utils "databasus-backend/internal/util/testing"+ "databasus-backend/internal/util/tools"+)++func Test_BackupShouldFailNotHang_WhenSaveFileFails_RegressionForIssue582(t *testing.T) {+ env := config.GetEnv()++ if env.TestMinioPort == "" {+ t.Skip("TEST_MINIO_PORT not set; skipping flaky-S3 regression test")+ return+ }++ router := createTestRouter()+ user := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace(+ "Issue 582 Workspace",+ user,+ router,+ )+ defer workspaces_testing.RemoveTestWorkspace(workspace, router)++ minioEndpoint := fmt.Sprintf("http://%s:%s", env.TestLocalhost, env.TestMinioPort)+ storage := storages.CreateTestFlakyS3Storage(workspace.ID, minioEndpoint)+ defer storages.RemoveTestStorage(storage.ID)++ t.Run("MariaDB", func(t *testing.T) {+ container, err := connectToMariadbContainer(+ tools.MariadbVersion1011,+ env.TestMariadb1011Port,+ )+ if err != nil {+ t.Skipf("Skipping: failed to connect to MariaDB test container: %v", err)+ return+ }+ defer container.DB.Close()++ setupMariadbTestData(t, container.DB)++ database := createMariadbDatabaseViaAPI(+ t, router, "Issue 582 MariaDB DB", workspace.ID,+ container.Host, container.Port,+ container.Username, container.Password, container.Database,+ container.Version,+ user.Token,+ )++ assertBackupFailsWithoutHanging(t, router, user.Token, database.ID, storage.ID)+ })++ t.Run("PostgreSQL", func(t *testing.T) {+ container, err := connectToPostgresContainer("16", env.TestPostgres16Port)+ if err != nil {+ t.Skipf("Skipping: failed to connect to PostgreSQL test container: %v", err)+ return+ }+ defer container.DB.Close()++ _, err = container.DB.Exec(createAndFillTableQuery("test_data"))+ require.NoError(t, err)++ database := createDatabaseViaAPI(+ t, router, "Issue 582 PostgreSQL DB", workspace.ID,+ container.Host, container.Port,+ container.Username, container.Password, container.Database,+ user.Token,+ )++ assertBackupFailsWithoutHanging(t, router, user.Token, database.ID, storage.ID)+ })++ t.Run("MySQL", func(t *testing.T) {+ container, err := connectToMysqlContainer(+ tools.MysqlVersion80,+ env.TestMysql80Port,+ )+ if err != nil {+ t.Skipf("Skipping: failed to connect to MySQL test container: %v", err)+ return+ }+ defer container.DB.Close()++ setupMysqlTestData(t, container.DB)++ database := createMysqlDatabaseViaAPI(+ t, router, "Issue 582 MySQL DB", workspace.ID,+ container.Host, container.Port,+ container.Username, container.Password, container.Database,+ container.Version,+ user.Token,+ )++ assertBackupFailsWithoutHanging(t, router, user.Token, database.ID, storage.ID)+ })++ t.Run("MongoDB", func(t *testing.T) {+ container, err := connectToMongodbContainer(+ tools.MongodbVersion7,+ env.TestMongodb70Port,+ )+ if err != nil {+ t.Skipf("Skipping: failed to connect to MongoDB test container: %v", err)+ return+ }+ defer func() { _ = container.Client.Disconnect(t.Context()) }()++ setupMongodbTestData(t, container)++ database := createMongodbDatabaseViaAPI(+ t, router, "Issue 582 MongoDB DB", workspace.ID,+ container.Host, container.Port,+ container.Username, container.Password, container.Database,+ container.AuthDatabase,+ container.Version,+ user.Token,+ )++ assertBackupFailsWithoutHanging(t, router, user.Token, database.ID, storage.ID)+ })+}++func assertBackupFailsWithoutHanging(+ t *testing.T,+ router *gin.Engine,+ token string,+ databaseID uuid.UUID,+ storageID uuid.UUID,+) {+ t.Helper()++ defer test_utils.MakeDeleteRequest(+ t,+ router,+ "/api/v1/databases/"+databaseID.String(),+ "Bearer "+token,+ http.StatusNoContent,+ )++ enableBackupsViaAPI(+ t, router, databaseID, storageID,+ backups_config.BackupEncryptionNone, token,+ )++ createBackupViaAPI(t, router, databaseID, token)++ backup := waitForBackupTerminalStatus(t, router, databaseID, token, 2*time.Minute)++ require.Equalf(+ t,+ backups_core.BackupStatusFailed,+ backup.Status,+ "issue #582: backup should be marked Failed when SaveFile fails; got status=%s",+ backup.Status,+ )+ require.NotNil(+ t,+ backup.FailMessage,+ "issue #582: failed backup must carry a fail message describing the SaveFile error",+ )+}++func waitForBackupTerminalStatus(+ t *testing.T,+ router *gin.Engine,+ databaseID uuid.UUID,+ token string,+ timeout time.Duration,+) *backups_core.Backup {+ deadline := time.Now().UTC().Add(timeout)+ pollInterval := 500 * time.Millisecond++ for time.Now().UTC().Before(deadline) {+ var response backups_dto.GetBackupsResponse+ test_utils.MakeGetRequestAndUnmarshal(+ t,+ router,+ fmt.Sprintf("/api/v1/backups?database_id=%s&limit=1", databaseID.String()),+ "Bearer "+token,+ http.StatusOK,+ &response,+ )++ if len(response.Backups) > 0 {+ b := response.Backups[0]+ if b.Status == backups_core.BackupStatusCompleted ||+ b.Status == backups_core.BackupStatusFailed ||+ b.Status == backups_core.BackupStatusCanceled {+ return b+ }+ }++ time.Sleep(pollInterval)+ }++ t.Fatalf(+ "backup for database %s did not reach a terminal status within %v "++ "(issue #582: backup hangs forever when SaveFile fails)",+ databaseID,+ timeout,+ )++ return nil+}