Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): prevent hang from SaveFile failur if error happens early

Rostislav Dugine465205fcb9d
diff --git a/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go
index 515f785e..9f192109 100644
--- a/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go
@@ -228,6 +228,10 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
backup.FileName,
storageReader,
)
+ if saveErr != nil {
+ _ = storageReader.CloseWithError(saveErr)
+ cancel()
+ }
saveErrCh <- saveErr
}()
@@ -252,6 +256,17 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
bytesWritten := <-bytesWrittenCh
waitErr := cmd.Wait()
+ select {
+ case earlySaveErr := <-saveErrCh:
+ if earlySaveErr != nil {
+ _ = zstdWriter.Close()
+ _ = uc.closeWriters(encryptionWriter, storageWriter)
+ return nil, fmt.Errorf("save to storage: %w", earlySaveErr)
+ }
+ saveErrCh <- nil
+ default:
+ }
+
select {
case <-ctx.Done():
uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)
diff --git a/backend/internal/features/backups/backups/usecases/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/mongodb/create_backup_uc.go
index 003a945f..c512a04b 100644
--- a/backend/internal/features/backups/backups/usecases/mongodb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/mongodb/create_backup_uc.go
@@ -190,6 +190,10 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
backup.FileName,
storageReader,
)
+ if saveErr != nil {
+ _ = storageReader.CloseWithError(saveErr)
+ cancel()
+ }
saveErrCh <- saveErr
}()
@@ -214,6 +218,16 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
bytesWritten := <-bytesWrittenCh
waitErr := cmd.Wait()
+ select {
+ case earlySaveErr := <-saveErrCh:
+ if earlySaveErr != nil {
+ _ = uc.closeWriters(encryptionWriter, storageWriter)
+ return nil, fmt.Errorf("save to storage: %w", earlySaveErr)
+ }
+ saveErrCh <- nil
+ default:
+ }
+
select {
case <-ctx.Done():
uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)
diff --git a/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go
index e5770c00..555eec69 100644
--- a/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go
@@ -247,6 +247,10 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
backup.FileName,
storageReader,
)
+ if saveErr != nil {
+ _ = storageReader.CloseWithError(saveErr)
+ cancel()
+ }
saveErrCh <- saveErr
}()
@@ -271,6 +275,17 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
bytesWritten := <-bytesWrittenCh
waitErr := cmd.Wait()
+ select {
+ case earlySaveErr := <-saveErrCh:
+ if earlySaveErr != nil {
+ _ = zstdWriter.Close()
+ _ = uc.closeWriters(encryptionWriter, storageWriter)
+ return nil, fmt.Errorf("save to storage: %w", earlySaveErr)
+ }
+ saveErrCh <- nil
+ default:
+ }
+
select {
case <-ctx.Done():
uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)
diff --git a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go
index 52062395..19556425 100644
--- a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go
@@ -192,6 +192,10 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
backup.FileName,
storageReader,
)
+ if saveErr != nil {
+ _ = storageReader.CloseWithError(saveErr)
+ cancel()
+ }
saveErrCh <- saveErr
}()
@@ -218,6 +222,16 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
bytesWritten := <-bytesWrittenCh
waitErr := cmd.Wait()
+ select {
+ case earlySaveErr := <-saveErrCh:
+ if earlySaveErr != nil {
+ _ = uc.closeWriters(encryptionWriter, storageWriter)
+ return nil, fmt.Errorf("save to storage: %w", earlySaveErr)
+ }
+ saveErrCh <- nil
+ default:
+ }
+
select {
case <-ctx.Done():
uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)
diff --git a/backend/internal/features/storages/testing.go b/backend/internal/features/storages/testing.go
index 768e2875..67768241 100644
--- a/backend/internal/features/storages/testing.go
+++ b/backend/internal/features/storages/testing.go
@@ -4,6 +4,8 @@ import (
"github.com/google/uuid"
local_storage "databasus-backend/internal/features/storages/models/local"
+ s3_storage "databasus-backend/internal/features/storages/models/s3"
+ "databasus-backend/internal/util/encryption"
)
func CreateTestStorage(workspaceID uuid.UUID) *Storage {
@@ -22,6 +24,40 @@ func CreateTestStorage(workspaceID uuid.UUID) *Storage {
return storage
}
+func CreateTestFlakyS3Storage(workspaceID uuid.UUID, endpoint string) *Storage {
+ encryptor := encryption.GetFieldEncryptor()
+
+ accessKey, err := encryptor.Encrypt("issue-582-access")
+ if err != nil {
+ panic(err)
+ }
+
+ secretKey, err := encryptor.Encrypt("issue-582-secret")
+ if err != nil {
+ panic(err)
+ }
+
+ storage := &Storage{
+ WorkspaceID: workspaceID,
+ Type: StorageTypeS3,
+ Name: "Flaky S3 " + uuid.New().String(),
+ S3Storage: &s3_storage.S3Storage{
+ S3Bucket: "issue-582-no-such-bucket-" + uuid.New().String(),
+ S3Region: "us-east-1",
+ S3AccessKey: accessKey,
+ S3SecretKey: secretKey,
+ S3Endpoint: endpoint,
+ },
+ }
+
+ saved, err := storageRepository.Save(storage)
+ if err != nil {
+ panic(err)
+ }
+
+ return saved
+}
+
func RemoveTestStorage(id uuid.UUID) {
storage, err := storageRepository.FindByID(id)
if err != nil {
diff --git a/backend/internal/features/tests/backup_hang_edge_cases_test.go b/backend/internal/features/tests/backup_hang_edge_cases_test.go
new file mode 100644
index 00000000..6c247c83
--- /dev/null
+++ b/backend/internal/features/tests/backup_hang_edge_cases_test.go
@@ -0,0 +1,223 @@
+package tests
+
+import (
+ "fmt"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ _ "github.com/go-sql-driver/mysql"
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/config"
+ backups_core "databasus-backend/internal/features/backups/backups/core"
+ backups_dto "databasus-backend/internal/features/backups/backups/dto"
+ backups_config "databasus-backend/internal/features/backups/config"
+ "databasus-backend/internal/features/storages"
+ users_enums "databasus-backend/internal/features/users/enums"
+ users_testing "databasus-backend/internal/features/users/testing"
+ workspaces_testing "databasus-backend/internal/features/workspaces/testing"
+ test_utils "databasus-backend/internal/util/testing"
+ "databasus-backend/internal/util/tools"
+)
+
+func Test_BackupShouldFailNotHang_WhenSaveFileFails_RegressionForIssue582(t *testing.T) {
+ env := config.GetEnv()
+
+ if env.TestMinioPort == "" {
+ t.Skip("TEST_MINIO_PORT not set; skipping flaky-S3 regression test")
+ return
+ }
+
+ router := createTestRouter()
+ user := users_testing.CreateTestUser(users_enums.UserRoleMember)
+ workspace := workspaces_testing.CreateTestWorkspace(
+ "Issue 582 Workspace",
+ user,
+ router,
+ )
+ defer workspaces_testing.RemoveTestWorkspace(workspace, router)
+
+ minioEndpoint := fmt.Sprintf("http://%s:%s", env.TestLocalhost, env.TestMinioPort)
+ storage := storages.CreateTestFlakyS3Storage(workspace.ID, minioEndpoint)
+ defer storages.RemoveTestStorage(storage.ID)
+
+ t.Run("MariaDB", func(t *testing.T) {
+ container, err := connectToMariadbContainer(
+ tools.MariadbVersion1011,
+ env.TestMariadb1011Port,
+ )
+ if err != nil {
+ t.Skipf("Skipping: failed to connect to MariaDB test container: %v", err)
+ return
+ }
+ defer container.DB.Close()
+
+ setupMariadbTestData(t, container.DB)
+
+ database := createMariadbDatabaseViaAPI(
+ t, router, "Issue 582 MariaDB DB", workspace.ID,
+ container.Host, container.Port,
+ container.Username, container.Password, container.Database,
+ container.Version,
+ user.Token,
+ )
+
+ assertBackupFailsWithoutHanging(t, router, user.Token, database.ID, storage.ID)
+ })
+
+ t.Run("PostgreSQL", func(t *testing.T) {
+ container, err := connectToPostgresContainer("16", env.TestPostgres16Port)
+ if err != nil {
+ t.Skipf("Skipping: failed to connect to PostgreSQL test container: %v", err)
+ return
+ }
+ defer container.DB.Close()
+
+ _, err = container.DB.Exec(createAndFillTableQuery("test_data"))
+ require.NoError(t, err)
+
+ database := createDatabaseViaAPI(
+ t, router, "Issue 582 PostgreSQL DB", workspace.ID,
+ container.Host, container.Port,
+ container.Username, container.Password, container.Database,
+ user.Token,
+ )
+
+ assertBackupFailsWithoutHanging(t, router, user.Token, database.ID, storage.ID)
+ })
+
+ t.Run("MySQL", func(t *testing.T) {
+ container, err := connectToMysqlContainer(
+ tools.MysqlVersion80,
+ env.TestMysql80Port,
+ )
+ if err != nil {
+ t.Skipf("Skipping: failed to connect to MySQL test container: %v", err)
+ return
+ }
+ defer container.DB.Close()
+
+ setupMysqlTestData(t, container.DB)
+
+ database := createMysqlDatabaseViaAPI(
+ t, router, "Issue 582 MySQL DB", workspace.ID,
+ container.Host, container.Port,
+ container.Username, container.Password, container.Database,
+ container.Version,
+ user.Token,
+ )
+
+ assertBackupFailsWithoutHanging(t, router, user.Token, database.ID, storage.ID)
+ })
+
+ t.Run("MongoDB", func(t *testing.T) {
+ container, err := connectToMongodbContainer(
+ tools.MongodbVersion7,
+ env.TestMongodb70Port,
+ )
+ if err != nil {
+ t.Skipf("Skipping: failed to connect to MongoDB test container: %v", err)
+ return
+ }
+ defer func() { _ = container.Client.Disconnect(t.Context()) }()
+
+ setupMongodbTestData(t, container)
+
+ database := createMongodbDatabaseViaAPI(
+ t, router, "Issue 582 MongoDB DB", workspace.ID,
+ container.Host, container.Port,
+ container.Username, container.Password, container.Database,
+ container.AuthDatabase,
+ container.Version,
+ user.Token,
+ )
+
+ assertBackupFailsWithoutHanging(t, router, user.Token, database.ID, storage.ID)
+ })
+}
+
+func assertBackupFailsWithoutHanging(
+ t *testing.T,
+ router *gin.Engine,
+ token string,
+ databaseID uuid.UUID,
+ storageID uuid.UUID,
+) {
+ t.Helper()
+
+ defer test_utils.MakeDeleteRequest(
+ t,
+ router,
+ "/api/v1/databases/"+databaseID.String(),
+ "Bearer "+token,
+ http.StatusNoContent,
+ )
+
+ enableBackupsViaAPI(
+ t, router, databaseID, storageID,
+ backups_config.BackupEncryptionNone, token,
+ )
+
+ createBackupViaAPI(t, router, databaseID, token)
+
+ backup := waitForBackupTerminalStatus(t, router, databaseID, token, 2*time.Minute)
+
+ require.Equalf(
+ t,
+ backups_core.BackupStatusFailed,
+ backup.Status,
+ "issue #582: backup should be marked Failed when SaveFile fails; got status=%s",
+ backup.Status,
+ )
+ require.NotNil(
+ t,
+ backup.FailMessage,
+ "issue #582: failed backup must carry a fail message describing the SaveFile error",
+ )
+}
+
+func waitForBackupTerminalStatus(
+ t *testing.T,
+ router *gin.Engine,
+ databaseID uuid.UUID,
+ token string,
+ timeout time.Duration,
+) *backups_core.Backup {
+ deadline := time.Now().UTC().Add(timeout)
+ pollInterval := 500 * time.Millisecond
+
+ for time.Now().UTC().Before(deadline) {
+ var response backups_dto.GetBackupsResponse
+ test_utils.MakeGetRequestAndUnmarshal(
+ t,
+ router,
+ fmt.Sprintf("/api/v1/backups?database_id=%s&limit=1", databaseID.String()),
+ "Bearer "+token,
+ http.StatusOK,
+ &response,
+ )
+
+ if len(response.Backups) > 0 {
+ b := response.Backups[0]
+ if b.Status == backups_core.BackupStatusCompleted ||
+ b.Status == backups_core.BackupStatusFailed ||
+ b.Status == backups_core.BackupStatusCanceled {
+ return b
+ }
+ }
+
+ time.Sleep(pollInterval)
+ }
+
+ t.Fatalf(
+ "backup for database %s did not reach a terminal status within %v "+
+ "(issue #582: backup hangs forever when SaveFile fails)",
+ databaseID,
+ timeout,
+ )
+
+ return nil
+}